mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Moving FileGroups instead of refreshing files after they have been moved.
This commit is contained in:
parent
6fe8a7b704
commit
0f232fd533
59 changed files with 1902 additions and 1040 deletions
|
|
@ -75,6 +75,14 @@ function handleEvents(events) {
|
|||
eventToast('Successful tag', message, 'success');
|
||||
}
|
||||
|
||||
if (event === 'file_move_completed') {
|
||||
eventToast('Successful Move', message, 'success');
|
||||
}
|
||||
|
||||
if (event === 'file_move_failed') {
|
||||
eventToast('Move Failed', message, 'error');
|
||||
}
|
||||
|
||||
if (subject) {
|
||||
newestEvents[subject] = dt;
|
||||
}
|
||||
|
|
|
|||
701
app/src/api.js
701
app/src/api.js
File diff suppressed because it is too large
Load diff
|
|
@ -118,13 +118,6 @@ function ArchivePage() {
|
|||
description: 'Archive update has been scheduled.',
|
||||
time: 2000,
|
||||
});
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Archive Downloading',
|
||||
description: 'Archive update failed.',
|
||||
time: 2000,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import React, {useState} from "react";
|
||||
import {Dimmer, Dropdown, Grid, Label, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
|
||||
import {Dimmer, Dropdown, Grid, Input, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
|
||||
import {
|
||||
createChannel,
|
||||
createChannelDownload,
|
||||
|
|
@ -19,6 +19,7 @@ import {
|
|||
frequencyOptions,
|
||||
HelpHeader,
|
||||
humanFileSize,
|
||||
humanNumber,
|
||||
RequiredAsterisk,
|
||||
SearchInput,
|
||||
secondsToFrequency,
|
||||
|
|
@ -77,6 +78,10 @@ function ChannelStatistics({statistics}) {
|
|||
<StatisticValue>{secondsToFullDuration(statistics.length)}</StatisticValue>
|
||||
<StatisticLabel>Total Duration</StatisticLabel>
|
||||
</Statistic>
|
||||
<Statistic>
|
||||
<StatisticValue>{humanNumber(statistics.video_tags)}</StatisticValue>
|
||||
<StatisticLabel>Video Tags</StatisticLabel>
|
||||
</Statistic>
|
||||
</Segment>
|
||||
}
|
||||
|
||||
|
|
@ -231,15 +236,7 @@ function ChannelPage({create, header}) {
|
|||
if (e) {
|
||||
e.preventDefault();
|
||||
}
|
||||
const response = await refreshChannel(channelId);
|
||||
if (response.status !== 204) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Failed to refresh',
|
||||
description: "Failed to refresh this channel's directory",
|
||||
time: 5000,
|
||||
})
|
||||
}
|
||||
await refreshChannel(channelId);
|
||||
}
|
||||
|
||||
const handleDelete = async () => {
|
||||
|
|
@ -295,19 +292,36 @@ function ChannelPage({create, header}) {
|
|||
>
|
||||
<ModalHeader>{channel.tag_name ? 'Modify Tag' : 'Add Tag'}</ModalHeader>
|
||||
<ModalContent>
|
||||
<TagsSelector
|
||||
limit={1}
|
||||
selectedTagNames={newTagName ? [newTagName] : []}
|
||||
onAdd={handleTagSelect}
|
||||
onRemove={() => handleTagSelect(null)}
|
||||
/>
|
||||
<br/>
|
||||
<Toggle
|
||||
label='Move to directory: '
|
||||
checked={moveToTagDirectory}
|
||||
onChange={setMoveToTagDirectory}
|
||||
/>
|
||||
{newTagDirectory && <Label size='large'>{newTagDirectory}</Label>}
|
||||
<Grid columns={1}>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<TagsSelector
|
||||
limit={1}
|
||||
selectedTagNames={newTagName ? [newTagName] : []}
|
||||
onAdd={handleTagSelect}
|
||||
onRemove={() => handleTagSelect(null)}
|
||||
/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Toggle
|
||||
label='Move to directory: '
|
||||
checked={moveToTagDirectory}
|
||||
onChange={setMoveToTagDirectory}
|
||||
/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Input fluid
|
||||
value={newTagDirectory}
|
||||
onChange={(e, {value}) => setNewTagDirectory(value)}
|
||||
disabled={!moveToTagDirectory}
|
||||
/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
</ModalContent>
|
||||
<ModalActions>
|
||||
<Button onClick={() => setTagModalOpen(false)}>Cancel</Button>
|
||||
|
|
@ -354,7 +368,6 @@ function ChannelPage({create, header}) {
|
|||
Directory <RequiredAsterisk/>
|
||||
</label>
|
||||
<DirectoryInput required
|
||||
disabled={create === undefined}
|
||||
value={channel.directory}
|
||||
setInput={value => changeValue('directory', value)}
|
||||
placeholder='videos/channel directory'
|
||||
|
|
@ -756,16 +769,21 @@ export function ChannelsPage() {
|
|||
</div>;
|
||||
|
||||
const headers = [
|
||||
{key: 'name', text: 'Name', sortBy: 'name', width: 8},
|
||||
{key: 'tag', text: 'Tag', sortBy: 'tag_name', width: 2},
|
||||
{key: 'video_count', text: 'Videos', sortBy: 'video_count', width: 2},
|
||||
{key: 'download_frequency', text: 'Download Frequency', sortBy: 'minimum_frequency', width: 2},
|
||||
{key: 'size', text: 'Size', sortBy: 'size', width: 2},
|
||||
{key: 'name', text: 'Name', sortBy: [i => i['name'].toLowerCase()], width: 8},
|
||||
{key: 'tag', text: 'Tag', sortBy: [i => i['tag_name'], i => i['name'].toLowerCase()], width: 2},
|
||||
{key: 'video_count', text: 'Videos', sortBy: [i => i['video_count'], i => i['name'].toLowerCase()], width: 2},
|
||||
{
|
||||
key: 'download_frequency',
|
||||
text: 'Download Frequency',
|
||||
sortBy: [i => i['minimum_frequency'], i => i['name'].toLowerCase()],
|
||||
width: 2
|
||||
},
|
||||
{key: 'size', text: 'Size', sortBy: [i => i['size'], i => i['name'].toLowerCase()], width: 2},
|
||||
{key: 'manage', text: 'Manage', width: 2},
|
||||
];
|
||||
const mobileHeaders = [
|
||||
{key: 'name', text: 'Name', sortBy: 'name'},
|
||||
{key: 'video_count', text: 'Videos', sortBy: 'video_count'},
|
||||
{key: 'name', text: 'Name', sortBy: [i => i['name'].toLowerCase()]},
|
||||
{key: 'video_count', text: 'Videos', sortBy: [i => i['video_count'], i => i['name'].toLowerCase()]},
|
||||
{key: 'manage', text: 'Manage'},
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ export const DEFAULT_LIMIT = 20;
|
|||
export const NAME = process.env && process.env.REACT_APP_NAME ? process.env.REACT_APP_NAME : null;
|
||||
|
||||
// Other services on a WROLPi.
|
||||
export const FILES_MEDIA_URI = `https://${window.location.hostname}:${window.location.port}/media/`;
|
||||
export const MAP_VIEWER_URI = `https://${window.location.hostname}:8084`;
|
||||
export const ZIM_VIEWER_URI = `https://${window.location.hostname}:8085`;
|
||||
export const HELP_VIEWER_URI = `https://${window.location.hostname}:8086`;
|
||||
|
|
@ -1281,7 +1282,10 @@ export function DirectoryInput({disabled, error, placeholder, setInput, value, r
|
|||
setDirectory(value);
|
||||
}
|
||||
|
||||
const {media_directory} = settings;
|
||||
let {media_directory} = settings;
|
||||
if (media_directory && !media_directory.endsWith('/')) {
|
||||
media_directory = `${media_directory}/`;
|
||||
}
|
||||
error = error || isDirectory && !isDir;
|
||||
|
||||
return <div>
|
||||
|
|
|
|||
|
|
@ -43,10 +43,8 @@ class Downloader extends React.Component {
|
|||
this.setState({pending: true, submitted: false});
|
||||
try {
|
||||
let response = await postDownload(urls, downloader, null, null, null, destination, tagNames);
|
||||
if (response.status === 204) {
|
||||
if (response.ok) {
|
||||
this.setState({urls: '', pending: false, submitted: true});
|
||||
} else {
|
||||
toast({type: 'error', title: 'Error', description: 'Failed to create download!'});
|
||||
}
|
||||
} finally {
|
||||
this.setState({pending: false});
|
||||
|
|
@ -216,8 +214,7 @@ class ChannelDownload extends React.Component {
|
|||
if (response.status === 204) {
|
||||
this.setState({pending: false, disabled: false, success: true, url: '', ready: false});
|
||||
} else {
|
||||
const content = await response.json();
|
||||
let error = content.message || null;
|
||||
let error = (await response.json()).message || null;
|
||||
this.setState({pending: false, disabled: false, success: false, error});
|
||||
}
|
||||
}
|
||||
|
|
@ -355,8 +352,7 @@ class RSSDownload extends ChannelDownload {
|
|||
if (response.status === 204) {
|
||||
this.setState({pending: false, disabled: false, success: true, url: '', ready: false});
|
||||
} else {
|
||||
const content = await response.json();
|
||||
let error = content.message || null;
|
||||
let error = (await response.json()).message || null;
|
||||
this.setState({pending: false, disabled: false, success: false, error});
|
||||
}
|
||||
}
|
||||
|
|
@ -487,15 +483,8 @@ class ScrapeDownloader extends Downloader {
|
|||
suffix,
|
||||
max_pages,
|
||||
);
|
||||
if (response.status === 204) {
|
||||
if (response.ok) {
|
||||
this.setState({urls: '', pending: false, submitted: true});
|
||||
} else {
|
||||
const content = await response.json();
|
||||
let error = 'Failed to create download!';
|
||||
if (content && content['error']) {
|
||||
error = content['error'];
|
||||
}
|
||||
toast({type: 'error', title: 'Error', description: error});
|
||||
}
|
||||
} finally {
|
||||
this.setState({pending: false});
|
||||
|
|
|
|||
|
|
@ -20,7 +20,9 @@ import {
|
|||
ErrorMessage,
|
||||
ExternalCardLink,
|
||||
FileIcon,
|
||||
FILES_MEDIA_URI,
|
||||
findPosterPath,
|
||||
HandPointMessage,
|
||||
humanFileSize,
|
||||
isoDatetimeToString,
|
||||
mimetypeColor,
|
||||
|
|
@ -583,6 +585,10 @@ function FilesPage() {
|
|||
return <>
|
||||
<FilesRefreshProgress/>
|
||||
<FileBrowser/>
|
||||
|
||||
<HandPointMessage>
|
||||
<p>You can also view your media directory at <a href={FILES_MEDIA_URI}>{FILES_MEDIA_URI}</a></p>
|
||||
</HandPointMessage>
|
||||
</>;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -504,35 +504,19 @@ class EditInventory extends React.Component {
|
|||
|
||||
handleSubmit = async (inventory, closeCallback) => {
|
||||
let inventoryId = inventory.id;
|
||||
delete inventory['id'];
|
||||
let response = await updateInventory(inventoryId, inventory);
|
||||
|
||||
if (response.status === 204) {
|
||||
if (response.ok) {
|
||||
await this.props.success();
|
||||
closeCallback();
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
header: 'Failed to save inventory!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
handleDelete = async (inventory, closeCallback) => {
|
||||
let response = await deleteInventory(inventory.id);
|
||||
|
||||
if (response.status === 204) {
|
||||
if (response.ok) {
|
||||
await this.props.success();
|
||||
closeCallback();
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
header: 'Failed to delete inventory!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -554,16 +538,9 @@ class NewInventory extends React.Component {
|
|||
delete inventory.id;
|
||||
let response = await saveInventory(inventory);
|
||||
|
||||
if (response.status === 201) {
|
||||
if (response.ok) {
|
||||
await this.props.success();
|
||||
closeCallback();
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
header: 'Failed to save inventory!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -953,18 +930,11 @@ class InventoryAddList extends React.Component {
|
|||
expiration_date: this.state.expiration_date,
|
||||
};
|
||||
let response = await saveItem(this.state.inventory.id, item);
|
||||
if (response.status === 204) {
|
||||
if (response.ok) {
|
||||
await this.clearInputs();
|
||||
await this.fetchItems();
|
||||
await this.fetchCategories();
|
||||
await this.fetchBrands();
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
header: 'Failed to save item!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,24 @@
|
|||
import React, {useState} from "react";
|
||||
import {Route, Routes, useNavigate} from "react-router-dom";
|
||||
import {Link, Route, Routes, useNavigate} from "react-router-dom";
|
||||
import {FilesSearchView} from "./Files";
|
||||
import {useLatestRequest, usePages, useSearchDate, useSearchFilter} from "../hooks/customHooks";
|
||||
import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks";
|
||||
import {ZimSearchView} from "./Zim";
|
||||
import {searchEstimateFiles, searchEstimateZims, searchSuggestions} from "../api";
|
||||
import {searchEstimateFiles, searchEstimateOthers, searchEstimateZims, searchSuggestions} from "../api";
|
||||
import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
|
||||
import _ from "lodash";
|
||||
import {TagsContext} from "../Tags";
|
||||
import {Button as SButton, Header as SHeader, Label} from "semantic-ui-react";
|
||||
import {Modal, ModalContent} from "./Theme";
|
||||
import {QueryContext} from "../contexts/contexts";
|
||||
import {
|
||||
AccordionContent,
|
||||
AccordionTitle,
|
||||
Button as SButton,
|
||||
Grid,
|
||||
GridColumn,
|
||||
GridRow,
|
||||
Header as SHeader,
|
||||
Label
|
||||
} from "semantic-ui-react";
|
||||
import {Accordion, Header, Icon, Loader, Modal, ModalContent, Segment} from "./Theme";
|
||||
import {QueryContext, ThemeContext} from "../contexts/contexts";
|
||||
|
||||
const SUGGESTED_APPS = [
|
||||
{location: '/admin', title: 'Downloads', description: 'View and control your downloads'},
|
||||
|
|
@ -128,6 +137,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
const {data: filesData, sendRequest: sendFilesRequest, loading: filesLoading} = useLatestRequest(500);
|
||||
// Zims are slow, so they are separate.
|
||||
const {data: zimData, sendRequest: sendZimRequest, loading: zimLoading} = useLatestRequest(500);
|
||||
const {data: otherData, sendRequest: sendOtherRequest, loading: otherLoading} = useLatestRequest(500);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (searchStr || (tagNames && tagNames.length > 0)) {
|
||||
|
|
@ -139,6 +149,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
}
|
||||
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
|
||||
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
|
||||
sendOtherRequest(async () => await searchEstimateOthers(tagNames));
|
||||
}
|
||||
}, [
|
||||
searchStr,
|
||||
|
|
@ -179,6 +190,14 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
}
|
||||
}, [setSuggestions, zimData]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!_.isEmpty(otherData)) {
|
||||
setSuggestions((prevState) => {
|
||||
return {...prevState, otherEstimates: otherData.others}
|
||||
});
|
||||
}
|
||||
}, [setSuggestions, otherData]);
|
||||
|
||||
return {suggestions, loading: generalLoading || zimLoading || filesLoading}
|
||||
}
|
||||
|
||||
|
|
@ -269,6 +288,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
}
|
||||
}
|
||||
|
||||
const otherSum = suggestions.otherEstimates ? _.sum(Object.values(suggestions.otherEstimates)) : 0;
|
||||
|
||||
const matchingApps = SUGGESTED_APPS.filter(i =>
|
||||
i.title.toLowerCase().includes(lowerSearchStr)
|
||||
|| fuzzyMatch(i.title.toLowerCase(), lowerSearchStr)
|
||||
|
|
@ -283,6 +304,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
setSuggestionsSums({
|
||||
fileGroups: newSuggestions.fileGroups,
|
||||
zims: zimSum,
|
||||
otherSum: otherSum,
|
||||
channels: newSuggestions.channels.length,
|
||||
domains: newSuggestions.domains.length,
|
||||
tags: matchingTags?.length,
|
||||
|
|
@ -342,10 +364,12 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
|
|||
|
||||
const filesTabName = <span>Files <Label>{normalizeEstimate(suggestionsSums?.fileGroups)}</Label></span>;
|
||||
const zimsTabName = <span>Zims <Label>{normalizeEstimate(suggestionsSums?.zims)}</Label></span>;
|
||||
const othersTabName = <span>Other <Label>{normalizeEstimate(suggestionsSums?.otherSum)}</Label></span>;
|
||||
|
||||
const links = [
|
||||
{text: filesTabName, to: '/search', key: 'filesSearch_', end: true},
|
||||
{text: zimsTabName, to: '/search/zim', key: 'zimsSearch'},
|
||||
{text: othersTabName, to: '/search/other', key: 'othersSearch'},
|
||||
];
|
||||
|
||||
return <React.Fragment>
|
||||
|
|
@ -353,6 +377,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
|
|||
<Routes>
|
||||
<Route path='/*' element={<FilesSearchView/>}/>
|
||||
<Route path='/zim' exact element={<ZimSearchView suggestions={suggestions} loading={loading}/>}/>
|
||||
<Route path='/other' exact element={<OtherSearchView loading={loading}/>}/>
|
||||
</Routes>
|
||||
</React.Fragment>
|
||||
}
|
||||
|
|
@ -403,3 +428,58 @@ export function SearchIconButton() {
|
|||
</Modal>
|
||||
</React.Fragment>
|
||||
}
|
||||
|
||||
function SearchChannelPreview({channel}) {
|
||||
const {t} = React.useContext(ThemeContext);
|
||||
|
||||
return <GridRow {...t}>
|
||||
<GridColumn>
|
||||
<Link to={`/videos/channel/${channel.id}/video`}>
|
||||
{channel.name}
|
||||
</Link>
|
||||
</GridColumn>
|
||||
</GridRow>
|
||||
}
|
||||
|
||||
function OtherSearchView({loading}) {
|
||||
const {searchParams} = React.useContext(QueryContext);
|
||||
const [activeIndex, setActiveIndex] = React.useState(0);
|
||||
const activeTags = searchParams.getAll('tag');
|
||||
const {channels, loading: channelsLoading} = useSearchChannels(activeTags);
|
||||
|
||||
const handleClick = (newIndex) => {
|
||||
console.log('newIndex', newIndex, 'activeIndex', activeIndex);
|
||||
setActiveIndex(activeIndex === newIndex ? null : newIndex);
|
||||
}
|
||||
|
||||
if (loading || channelsLoading) {
|
||||
return <Accordion>
|
||||
<Segment><Loader active/></Segment>
|
||||
</Accordion>
|
||||
}
|
||||
|
||||
const channelsAccordion = <React.Fragment>
|
||||
<AccordionTitle
|
||||
index={0}
|
||||
active={activeIndex === 0}
|
||||
onClick={() => handleClick(0)}
|
||||
>
|
||||
<Header as='h3'>
|
||||
<Icon name='dropdown'/>
|
||||
Channels
|
||||
<Label>{normalizeEstimate(channels.length)}</Label>
|
||||
</Header>
|
||||
</AccordionTitle>
|
||||
<AccordionContent active={activeIndex === 0}>
|
||||
<Grid>
|
||||
{!_.isEmpty(channels) ?
|
||||
channels.map(i => <SearchChannelPreview channel={i}/>)
|
||||
: 'No Channels'}
|
||||
</Grid>
|
||||
</AccordionContent>
|
||||
</React.Fragment>;
|
||||
|
||||
return <Accordion>
|
||||
{channelsAccordion}
|
||||
</Accordion>
|
||||
}
|
||||
|
|
@ -243,6 +243,7 @@ function VideosStatistics() {
|
|||
{key: 'month', label: 'Downloads Past Month'},
|
||||
{key: 'year', label: 'Downloads Past Year'},
|
||||
{key: 'sum_duration', label: 'Total Duration'},
|
||||
{key: 'censored_videos', label: 'Censored Videos'},
|
||||
];
|
||||
const commentsNames = [
|
||||
{key: 'have_comments', label: 'Have Comments'},
|
||||
|
|
@ -255,6 +256,7 @@ function VideosStatistics() {
|
|||
];
|
||||
const channelNames = [
|
||||
{key: 'channels', label: 'Channels'},
|
||||
{key: 'tagged_channels', label: 'Tagged Channels'},
|
||||
];
|
||||
|
||||
const buildSegment = (title, names, stats) => {
|
||||
|
|
|
|||
|
|
@ -255,18 +255,7 @@ class RecurringDownloadRow extends React.Component {
|
|||
handleRestart = async () => {
|
||||
const {id} = this.props;
|
||||
try {
|
||||
const response = await restartDownload(id);
|
||||
if (response.status !== 204) {
|
||||
throw Error('Unable to restart download');
|
||||
}
|
||||
} catch (e) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Error',
|
||||
description: 'Unable to restart download',
|
||||
time: 5000,
|
||||
})
|
||||
throw e;
|
||||
await restartDownload(id);
|
||||
} finally {
|
||||
if (this.props.fetchDownloads) {
|
||||
this.props.fetchDownloads();
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ export class SettingsPage extends React.Component {
|
|||
qrOpen: false,
|
||||
ready: false,
|
||||
|
||||
archive_directory: null,
|
||||
archive_destination: null,
|
||||
download_on_startup: null,
|
||||
download_timeout: null,
|
||||
hotspot_device: null,
|
||||
|
|
@ -190,12 +190,12 @@ export class SettingsPage extends React.Component {
|
|||
hotspot_status: null,
|
||||
ignore_outdated_zims: null,
|
||||
log_level: null,
|
||||
map_directory: null,
|
||||
map_destination: null,
|
||||
navColor: null,
|
||||
throttle_on_startup: null,
|
||||
throttle_status: null,
|
||||
videos_directory: null,
|
||||
zims_directory: null,
|
||||
videos_destination: null,
|
||||
zims_destination: null,
|
||||
}
|
||||
|
||||
this.handleSubmit = this.handleSubmit.bind(this);
|
||||
|
|
@ -207,7 +207,7 @@ export class SettingsPage extends React.Component {
|
|||
const settings = await getSettings();
|
||||
this.setState({
|
||||
ready: true,
|
||||
archive_directory: settings.archive_directory,
|
||||
archive_destination: settings.archive_destination,
|
||||
disabled: settings.wrol_mode,
|
||||
download_on_startup: settings.download_on_startup,
|
||||
download_timeout: settings.download_timeout || '',
|
||||
|
|
@ -217,12 +217,12 @@ export class SettingsPage extends React.Component {
|
|||
hotspot_status: settings.hotspot_status,
|
||||
ignore_outdated_zims: settings.ignore_outdated_zims,
|
||||
log_level: fromApiLogLevel(settings.log_level),
|
||||
map_directory: settings.map_directory,
|
||||
map_destination: settings.map_destination,
|
||||
navColor: settings.nav_color || 'violet',
|
||||
throttle_on_startup: settings.throttle_on_startup,
|
||||
throttle_status: settings.throttle_status,
|
||||
videos_directory: settings.videos_directory,
|
||||
zims_directory: settings.zims_directory,
|
||||
videos_destination: settings.videos_destination,
|
||||
zims_destination: settings.zims_destination,
|
||||
}, this.handleHotspotChange);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
|
|
@ -233,7 +233,7 @@ export class SettingsPage extends React.Component {
|
|||
async handleSubmit(callback) {
|
||||
this.setState({disabled: true, pending: true});
|
||||
let settings = {
|
||||
archive_directory: this.state.archive_directory,
|
||||
archive_destination: this.state.archive_destination,
|
||||
download_on_startup: this.state.download_on_startup,
|
||||
download_timeout: this.state.download_timeout ? parseInt(this.state.download_timeout) : 0,
|
||||
hotspot_device: this.state.hotspot_device,
|
||||
|
|
@ -241,11 +241,11 @@ export class SettingsPage extends React.Component {
|
|||
hotspot_ssid: this.state.hotspot_ssid,
|
||||
ignore_outdated_zims: this.state.ignore_outdated_zims,
|
||||
log_level: toApiLogLevel(this.state.log_level),
|
||||
map_directory: this.state.map_directory,
|
||||
map_destination: this.state.map_destination,
|
||||
nav_color: this.state.navColor,
|
||||
throttle_on_startup: this.state.throttle_on_startup,
|
||||
videos_directory: this.state.videos_directory,
|
||||
zims_directory: this.state.zims_directory,
|
||||
videos_destination: this.state.videos_destination,
|
||||
zims_destination: this.state.zims_destination,
|
||||
}
|
||||
try {
|
||||
const response = await saveSettings(settings);
|
||||
|
|
@ -304,7 +304,7 @@ export class SettingsPage extends React.Component {
|
|||
}
|
||||
|
||||
const {
|
||||
archive_directory,
|
||||
archive_destination,
|
||||
disabled,
|
||||
download_on_startup,
|
||||
download_timeout,
|
||||
|
|
@ -314,13 +314,13 @@ export class SettingsPage extends React.Component {
|
|||
hotspot_ssid,
|
||||
ignore_outdated_zims,
|
||||
log_level,
|
||||
map_directory,
|
||||
map_destination,
|
||||
navColor,
|
||||
pending,
|
||||
qrCodeValue,
|
||||
throttle_on_startup,
|
||||
videos_directory,
|
||||
zims_directory,
|
||||
videos_destination,
|
||||
zims_destination,
|
||||
} = this.state;
|
||||
|
||||
const qrButton = <Button icon style={{marginBottom: '1em'}}><Icon name='qrcode' size='big'/></Button>;
|
||||
|
|
@ -468,9 +468,9 @@ export class SettingsPage extends React.Component {
|
|||
<label>Archive Directory</label>
|
||||
<Input
|
||||
label={mediaDirectoryLabel}
|
||||
value={archive_directory}
|
||||
value={archive_destination}
|
||||
disabled={!edit_special_directories}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'archive_directory', d.value)}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'archive_destination', d.value)}
|
||||
/>
|
||||
</FormField>
|
||||
</GridColumn>
|
||||
|
|
@ -479,9 +479,9 @@ export class SettingsPage extends React.Component {
|
|||
<label>Videos Directory</label>
|
||||
<Input
|
||||
label={mediaDirectoryLabel}
|
||||
value={videos_directory}
|
||||
value={videos_destination}
|
||||
disabled={!edit_special_directories}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'videos_directory', d.value)}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'videos_destination', d.value)}
|
||||
/>
|
||||
</FormField>
|
||||
</GridColumn>
|
||||
|
|
@ -492,9 +492,9 @@ export class SettingsPage extends React.Component {
|
|||
<label>Map Directory</label>
|
||||
<Input
|
||||
label={mediaDirectoryLabel}
|
||||
value={map_directory}
|
||||
value={map_destination}
|
||||
disabled={!edit_special_directories}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'map_directory', d.value)}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'map_destination', d.value)}
|
||||
/>
|
||||
</FormField>
|
||||
</GridColumn>
|
||||
|
|
@ -503,9 +503,9 @@ export class SettingsPage extends React.Component {
|
|||
<label>Zims Directory</label>
|
||||
<Input
|
||||
label={mediaDirectoryLabel}
|
||||
value={zims_directory}
|
||||
value={zims_destination}
|
||||
disabled={!edit_special_directories}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'zims_directory', d.value)}
|
||||
onChange={(e, d) => this.handleInputChange(e, 'zims_destination', d.value)}
|
||||
/>
|
||||
</FormField>
|
||||
</GridColumn>
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import {
|
|||
getStatus,
|
||||
getVideo,
|
||||
getVideosStatistics,
|
||||
searchArchives,
|
||||
searchArchives, searchChannels,
|
||||
searchDirectories,
|
||||
searchVideos,
|
||||
searchZim,
|
||||
|
|
@ -1230,4 +1230,31 @@ export const useDockerized = () => {
|
|||
export const useCalcQuery = () => {
|
||||
const [calc, setCalc] = useOneQuery('calc');
|
||||
return {calc, setCalc}
|
||||
}
|
||||
|
||||
export const useSearchChannels = (defaultTagNames) => {
|
||||
const [tagNames, setTagNames] = useState(defaultTagNames || []);
|
||||
const [channels, setChannels] = useState([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const localSearchChannels = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const {channels: newChannels} = await searchChannels(tagNames);
|
||||
setChannels(newChannels);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
localSearchChannels();
|
||||
}, [tagNames]);
|
||||
|
||||
return {
|
||||
tagNames,
|
||||
setTagNames,
|
||||
channels,
|
||||
loading,
|
||||
}
|
||||
}
|
||||
|
|
@ -49,17 +49,20 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
singlefile, readability, screenshot = await self.do_archive(download)
|
||||
|
||||
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
|
||||
archive_id = archive.id
|
||||
|
||||
if download.settings and (tag_names := download.settings.get('tag_names')):
|
||||
for name in tag_names:
|
||||
archive.add_tag(name)
|
||||
with get_db_session() as session:
|
||||
archive = Archive.find_by_id(archive_id, session)
|
||||
if download.settings and (tag_names := download.settings.get('tag_names')):
|
||||
for name in tag_names:
|
||||
archive.add_tag(name, session)
|
||||
|
||||
if session := Session.object_session(archive):
|
||||
session.commit()
|
||||
if session := Session.object_session(archive):
|
||||
session.commit()
|
||||
|
||||
logger.info(f'Successfully downloaded Archive {download.url} {archive}')
|
||||
logger.info(f'Successfully downloaded Archive {download.url} {archive}')
|
||||
|
||||
return DownloadResult(success=True, location=f'/archive/{archive.id}')
|
||||
return DownloadResult(success=True, location=f'/archive/{archive.id}')
|
||||
|
||||
@optional_session
|
||||
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
archive.validate()
|
||||
|
||||
for tag_name in tag_names:
|
||||
archive.add_tag(tag_name)
|
||||
archive.add_tag(tag_name, test_session)
|
||||
|
||||
return archive
|
||||
|
||||
|
|
|
|||
|
|
@ -205,7 +205,8 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
|
|||
archive.file_group.download_datetime = now()
|
||||
archive.url = url
|
||||
archive.domain = get_or_create_domain(session, url)
|
||||
session.flush()
|
||||
archive.flush()
|
||||
archive.domain.flush()
|
||||
|
||||
return archive
|
||||
|
||||
|
|
|
|||
|
|
@ -315,8 +315,8 @@ class Archive(Base, ModelHelper):
|
|||
archive = model_archive(file_group, session)
|
||||
return archive
|
||||
|
||||
def add_tag(self, tag_id_or_name: int | str) -> TagFile:
|
||||
return self.file_group.add_tag(tag_id_or_name)
|
||||
def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile:
|
||||
return self.file_group.add_tag(tag_id_or_name, session)
|
||||
|
||||
|
||||
class Domain(Base, ModelHelper):
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_
|
|||
@pytest.mark.asyncio
|
||||
async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory):
|
||||
"""Tagged Archives can be searched."""
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
archive_factory(domain='example.com', tag_names=[tag.name, ])
|
||||
test_session.commit()
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,15 @@ from http import HTTPStatus
|
|||
|
||||
from sanic import Blueprint
|
||||
from sanic.request import Request
|
||||
from sanic_ext import validate
|
||||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi.api_utils import json_response
|
||||
from wrolpi.common import logger
|
||||
from . import lib, schema
|
||||
from .channel.api import channel_bp
|
||||
from .lib import format_videos_destination
|
||||
from .models import Channel
|
||||
from .video.api import video_bp
|
||||
|
||||
content_bp = Blueprint('VideoContent', '/api/videos')
|
||||
|
|
@ -25,3 +28,20 @@ logger = logger.getChild(__name__)
|
|||
async def statistics(_: Request):
|
||||
ret = await lib.get_statistics()
|
||||
return json_response(ret, HTTPStatus.OK)
|
||||
|
||||
|
||||
@content_bp.post('/tag_info')
|
||||
@openapi.definition(
|
||||
description='Get data about tagging a Channel',
|
||||
body=schema.ChannelTagInfoRequest,
|
||||
)
|
||||
@validate(schema.ChannelTagInfoRequest)
|
||||
async def channel_tag_info(_: Request, body: schema.ChannelTagInfoRequest):
|
||||
from wrolpi.tags import Tag
|
||||
channel = Channel.find_by_id(body.channel_id) if body.channel_id else None
|
||||
channel_name = channel.name if channel else None
|
||||
channel_url = channel.url if channel else None
|
||||
tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
|
||||
videos_destination = format_videos_destination(channel_name, tag_name, channel_url)
|
||||
ret = dict(videos_destination=videos_destination)
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from wrolpi.vars import PYTEST
|
|||
from . import lib
|
||||
from .. import schema
|
||||
from ..errors import UnknownChannel
|
||||
from ..lib import save_channels_config, format_videos_destination
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Channel
|
||||
|
||||
channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels')
|
||||
|
|
@ -78,9 +78,9 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
|
|||
@validate(schema.ChannelPutRequest)
|
||||
@run_after(save_channels_config)
|
||||
@wrol_mode_check
|
||||
def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
|
||||
async def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
|
||||
body.directory = pathlib.Path(body.directory) if body.directory else None
|
||||
channel = lib.update_channel(data=body, channel_id=channel_id)
|
||||
channel = await lib.update_channel(data=body, channel_id=channel_id)
|
||||
return response.raw('', HTTPStatus.NO_CONTENT,
|
||||
headers={'Location': f'/api/videos/channels/{channel.id}'})
|
||||
|
||||
|
|
@ -138,32 +138,14 @@ async def channel_download(_: Request, channel_id: int, body: schema.ChannelDown
|
|||
return response.empty()
|
||||
|
||||
|
||||
@channel_bp.post('/<channel_id:int>/tag_info')
|
||||
@openapi.definition(
|
||||
description='Get Channel Tag Info',
|
||||
body=schema.ChannelTagInfoRequest,
|
||||
)
|
||||
@validate(schema.ChannelTagInfoRequest)
|
||||
@run_after(save_channels_config)
|
||||
@wrol_mode_check
|
||||
async def channel_tag_info(_: Request, channel_id: int, body: schema.ChannelTagInfoRequest):
|
||||
from wrolpi.tags import Tag
|
||||
channel_name = Channel.find_by_id(channel_id).name
|
||||
tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
|
||||
videos_destination = format_videos_destination(channel_name, tag_name)
|
||||
ret = dict(videos_destination=videos_destination)
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
@channel_bp.post('/<channel_id:int>/tag')
|
||||
@openapi.definition(
|
||||
description='Tag Channel',
|
||||
description='Tag/untag a Channel with a single tag',
|
||||
body=schema.ChannelTagRequest,
|
||||
)
|
||||
@openapi.response(HTTPStatus.NO_CONTENT)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
@validate(schema.ChannelTagRequest)
|
||||
@run_after(save_channels_config)
|
||||
@wrol_mode_check
|
||||
async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagRequest):
|
||||
directory = None
|
||||
|
|
@ -174,3 +156,17 @@ async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagReques
|
|||
await lib.tag_channel(body.tag_name, directory, channel_id)
|
||||
|
||||
return response.raw('', HTTPStatus.NO_CONTENT)
|
||||
|
||||
|
||||
@channel_bp.post('/search')
|
||||
@openapi.definition(
|
||||
description='Search Channels',
|
||||
body=schema.ChannelSearchRequest,
|
||||
)
|
||||
@validate(schema.ChannelSearchRequest)
|
||||
async def channel_search(_: Request, body: schema.ChannelSearchRequest):
|
||||
channels = await lib.search_channels(body.tag_names)
|
||||
ret = dict(
|
||||
channels=channels,
|
||||
)
|
||||
return json_response(ret)
|
||||
|
|
@ -7,10 +7,11 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import run_after, logger, \
|
||||
get_media_directory, wrol_mode_check
|
||||
get_media_directory, wrol_mode_check, background_task
|
||||
from wrolpi.db import get_db_curs, optional_session, get_db_session
|
||||
from wrolpi.downloader import save_downloads_config, download_manager
|
||||
from wrolpi.errors import UnknownDirectory, APIError, ValidationError, RefreshConflict
|
||||
from wrolpi.errors import APIError, ValidationError, RefreshConflict
|
||||
from wrolpi.vars import PYTEST
|
||||
from .. import schema
|
||||
from ..common import check_for_channel_conflicts
|
||||
from ..errors import UnknownChannel, ChannelDirectoryConflict
|
||||
|
|
@ -115,22 +116,10 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
|
|||
|
||||
@run_after(save_channels_config)
|
||||
@optional_session
|
||||
def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
|
||||
async def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
|
||||
"""Update a Channel's DB record"""
|
||||
channel = Channel.find_by_id(channel_id, session)
|
||||
|
||||
# Only update directory if it was empty
|
||||
if data.directory and not channel.directory:
|
||||
data.directory = get_media_directory() / data.directory
|
||||
if not data.directory.is_dir():
|
||||
if data.mkdir:
|
||||
data.directory.mkdir()
|
||||
else:
|
||||
raise UnknownDirectory()
|
||||
elif data.directory:
|
||||
# Keep the old directory because the user can't change a Channel's directory.
|
||||
data.directory = channel.directory
|
||||
|
||||
# Verify that the URL/Name/directory aren't taken
|
||||
check_for_channel_conflicts(
|
||||
session,
|
||||
|
|
@ -140,11 +129,27 @@ def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_
|
|||
directory=data.directory,
|
||||
)
|
||||
|
||||
data = data.__dict__
|
||||
old_directory = pathlib.Path(channel.directory)
|
||||
new_directory = get_media_directory() / data.pop('directory')
|
||||
|
||||
# Apply the changes now that we've OK'd them
|
||||
channel.update(data.__dict__)
|
||||
channel.update(data)
|
||||
|
||||
session.commit()
|
||||
|
||||
async def _():
|
||||
await channel.move_channel(new_directory, session, send_events=True)
|
||||
|
||||
if new_directory != old_directory:
|
||||
if not new_directory.is_dir():
|
||||
new_directory.mkdir(parents=True)
|
||||
|
||||
if PYTEST:
|
||||
await _()
|
||||
else:
|
||||
background_task(_())
|
||||
|
||||
return channel
|
||||
|
||||
|
||||
|
|
@ -252,7 +257,7 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
|
|||
|
||||
@wrol_mode_check
|
||||
@optional_session
|
||||
async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: int, session: Session = None):
|
||||
async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, channel_id: int, session: Session = None):
|
||||
"""Add a Tag to a Channel, or remove a Tag from a Channel if no `tag_name` is provided.
|
||||
|
||||
Move the Channel to the new directory, if provided."""
|
||||
|
|
@ -261,29 +266,39 @@ async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id:
|
|||
if directory and flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to move channel while file refresh is in progress')
|
||||
|
||||
channel = Channel.find_by_id(channel_id, session=session)
|
||||
channel = Channel.find_by_id(channel_id, session)
|
||||
|
||||
if tag_name:
|
||||
tag = Tag.find_by_name(tag_name, session=session)
|
||||
tag = Tag.find_by_name(tag_name, session)
|
||||
channel.tag_id = tag.id
|
||||
channel.tag = tag
|
||||
else:
|
||||
channel.tag = channel.tag_id = None
|
||||
|
||||
session.flush([channel, ])
|
||||
channel.flush()
|
||||
session.commit()
|
||||
|
||||
# Do not move Channel files into a directory with files.
|
||||
if directory and directory.is_dir() and next(directory.iterdir(), None):
|
||||
if directory and directory != channel.directory and directory.is_dir() and next(directory.iterdir(), None):
|
||||
raise ChannelDirectoryConflict('Channel directory already exists and is not empty')
|
||||
|
||||
# Only move Channel when requested.
|
||||
if directory:
|
||||
# Move to newly defined directory only if necessary.
|
||||
logger.info(f'Tagging {channel} with {tag_name} moving to {directory} from {channel.directory}')
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
if channel.directory != directory:
|
||||
await channel.move(directory, session)
|
||||
coro = channel.move_channel(directory, session, send_events=True)
|
||||
if PYTEST:
|
||||
await coro
|
||||
else:
|
||||
background_task(coro)
|
||||
else:
|
||||
logger.info(f'Tagging {channel} with {tag_name}')
|
||||
|
||||
session.commit()
|
||||
|
||||
@optional_session
|
||||
async def search_channels(tag_names: List[str], session: Session) -> List[Channel]:
|
||||
"""Search Tagged Channels."""
|
||||
from wrolpi.tags import Tag
|
||||
channels = session.query(Channel).join(Tag).filter(Tag.name.in_(tag_names)).all()
|
||||
return channels
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
|
|
@ -96,7 +97,8 @@ def test_video_file_name(test_session, simple_video, test_client):
|
|||
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
|
||||
|
||||
|
||||
def test_channel_conflicts(test_client, test_session, test_directory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_conflicts(test_async_client, test_session, test_directory):
|
||||
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
||||
pathlib.Path(channel_directory).mkdir()
|
||||
new_channel = dict(
|
||||
|
|
@ -105,11 +107,11 @@ def test_channel_conflicts(test_client, test_session, test_directory):
|
|||
url='https://example.com/channel1',
|
||||
)
|
||||
|
||||
def _post_channel(channel):
|
||||
return test_client.post('/api/videos/channels', content=json.dumps(channel))
|
||||
async def _post_channel(channel):
|
||||
return await test_async_client.post('/api/videos/channels', content=json.dumps(channel))
|
||||
|
||||
# Create it
|
||||
request, response = _post_channel(new_channel)
|
||||
request, response = await _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
|
||||
# Name is an exact match
|
||||
|
|
@ -119,7 +121,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
|
|||
directory=channel_directory2,
|
||||
name='Example Channel 1',
|
||||
)
|
||||
request, response = _post_channel(new_channel)
|
||||
request, response = await _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json == {'cause': {'code': 'CHANNEL_NAME_CONFLICT',
|
||||
'error': 'The channel name is already taken.',
|
||||
|
|
@ -133,7 +135,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
|
|||
directory=channel_directory,
|
||||
name='name is fine',
|
||||
)
|
||||
request, response = _post_channel(new_channel)
|
||||
request, response = await _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json == {'cause': {'code': 'CHANNEL_DIRECTORY_CONFLICT',
|
||||
'error': 'The directory is already used by another channel.',
|
||||
|
|
@ -150,7 +152,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
|
|||
name='name is fine',
|
||||
url='https://example.com/channel1',
|
||||
)
|
||||
request, response = _post_channel(new_channel)
|
||||
request, response = await _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json == {'cause': {'code': 'CHANNEL_URL_CONFLICT',
|
||||
'error': 'The URL is already used by another channel.',
|
||||
|
|
@ -384,7 +386,8 @@ def test_search_videos_channel(test_client, test_session, video_factory):
|
|||
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
||||
|
||||
|
||||
def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
|
||||
for i in range(50):
|
||||
video_factory(channel_id=simple_channel.id)
|
||||
|
||||
|
|
@ -406,8 +409,9 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
|
|||
]
|
||||
last_ids = []
|
||||
for offset, video_count in tests:
|
||||
_, response = assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', offset=offset,
|
||||
limit=20)
|
||||
_, response = await assert_video_search(channel_id=simple_channel.id, order_by='published_datetime',
|
||||
offset=offset,
|
||||
limit=20)
|
||||
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
|
||||
current_ids = [i['id'] for i in response.json['file_groups']]
|
||||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
|
|
@ -416,7 +420,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory):
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
|
||||
# Create Download.
|
||||
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
|
||||
|
|
@ -465,7 +469,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
|
|||
videos_directory = get_videos_directory()
|
||||
channel = channel_factory(name='Channel Name', download_frequency=120)
|
||||
channel_directory = channel.directory
|
||||
tag = tag_factory('Tag Name')
|
||||
tag = await tag_factory('Tag Name')
|
||||
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
|
||||
# Create recurring download which uses the Channel's directory.
|
||||
download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
|
||||
|
|
@ -532,7 +536,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
|
|||
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
|
||||
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
|
||||
|
||||
# Channel can be Tagged, but not move directories.
|
||||
# Channel can be Tagged, without moving directories.
|
||||
body = dict(tag_name=tag.name)
|
||||
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
|
||||
|
|
@ -545,3 +549,79 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
|
|||
d1, d2 = test_session.query(Download).all()
|
||||
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
|
||||
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory,
|
||||
video_factory, test_channels_config, test_download_manager, test_downloader):
|
||||
"""A Channel can be moved."""
|
||||
channel = channel_factory(name='Channel Name')
|
||||
vid1 = video_factory(channel_id=channel.id)
|
||||
vid2 = video_factory(channel_id=channel.id)
|
||||
test_session.commit()
|
||||
|
||||
assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
|
||||
assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
|
||||
assert str(vid2.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
|
||||
|
||||
body = dict(
|
||||
name=channel.name,
|
||||
directory=str(test_directory / 'videos/New Directory'),
|
||||
)
|
||||
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
|
||||
channel = Channel.find_by_id(channel.id)
|
||||
assert str(channel.directory) == str(test_directory / 'videos/New Directory')
|
||||
assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/'))
|
||||
assert str(vid2.video_path).startswith(str(test_directory / 'videos/New Directory/'))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_channel_after_terminal_move(test_async_client, test_session, test_directory, channel_factory,
|
||||
tag_factory, video_factory, test_channels_config, test_download_manager,
|
||||
test_downloader):
|
||||
"""A Channel can be moved in the UI after being moved in the Terminal."""
|
||||
channel = channel_factory(name='Channel Name')
|
||||
vid1 = video_factory(channel_id=channel.id)
|
||||
test_session.commit()
|
||||
|
||||
assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
|
||||
assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
|
||||
|
||||
# mv 'videos/Channel Name' 'videos/New Directory'
|
||||
shutil.move(channel.directory, test_directory / 'videos/New Directory')
|
||||
|
||||
body = dict(
|
||||
name=channel.name,
|
||||
directory=str(test_directory / 'videos/New Directory'),
|
||||
)
|
||||
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
|
||||
channel = Channel.find_by_id(channel.id)
|
||||
vid1 = test_session.query(Video).one()
|
||||
assert str(channel.directory) == str(test_directory / 'videos/New Directory')
|
||||
assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/'))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_tagged_channels(test_async_client, test_session, channel_factory, tag_factory,
|
||||
video_factory):
|
||||
"""Tagged Channels can be searched."""
|
||||
tag = await tag_factory()
|
||||
test_session.commit()
|
||||
|
||||
# Can search when empty.
|
||||
body = dict(tag_names=[tag.name])
|
||||
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['channels']) == 0, 'No Channels should be tagged'
|
||||
|
||||
channel1 = channel_factory(tag_name=tag.name)
|
||||
channel2 = channel_factory()
|
||||
test_session.commit()
|
||||
|
||||
body = dict(tag_names=[tag.name])
|
||||
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['channels']) == 1, 'Only one Channel is tagged'
|
||||
assert response.json['channels'][0]['id'] == channel1.id
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from typing import List
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from modules.videos import schema
|
||||
from modules.videos.channel import lib
|
||||
|
|
@ -124,3 +125,23 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact
|
|||
|
||||
await assert_search('foo', ['foo', 'foo bar', 'foobar'])
|
||||
await assert_search('bar', ['foobar', 'foo bar', 'bar'], True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_channel_existing(test_async_client, test_session, test_directory, channel_factory, tag_factory,
|
||||
video_factory, test_channels_config):
|
||||
"""A Channel can already exist in a Tag's directory and still be tagged."""
|
||||
channel_directory = test_directory / 'videos/one/Channel Name'
|
||||
channel = channel_factory(name='Channel Name', download_frequency=120, directory=channel_directory)
|
||||
video_factory(channel_id=channel.id)
|
||||
test_session.commit()
|
||||
assert str(channel.directory).startswith(f'{test_directory}/videos/one/')
|
||||
|
||||
tag = await tag_factory()
|
||||
await lib.tag_channel(tag.name, channel_directory, channel.id, test_session)
|
||||
|
||||
# Channel's tag is saved to the config.
|
||||
save_channels_config(test_session)
|
||||
with test_channels_config.open() as fh:
|
||||
config = yaml.load(fh, Loader=yaml.Loader)
|
||||
assert config['channels'][0]['tag_name'] == tag.name
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from wrolpi.captions import FFMPEG_BIN
|
||||
from wrolpi.cmd import FFPROBE_BIN
|
||||
from wrolpi.common import logger, get_media_directory, get_wrolpi_config
|
||||
from wrolpi.common import logger, get_media_directory
|
||||
from wrolpi.dates import seconds_to_timestamp
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
|
||||
|
|
@ -204,7 +204,7 @@ def is_valid_poster(poster_path: Path) -> bool:
|
|||
async def ffprobe_json(video_path: Union[Path, str]) -> dict:
|
||||
"""Extract video file metadata using ffprobe.
|
||||
|
||||
>>> ffprobe_json('video')
|
||||
>>> ffprobe_json('video.mp4')
|
||||
{
|
||||
'chapters': [],
|
||||
'format': {},
|
||||
|
|
@ -270,39 +270,6 @@ def get_video_duration(video_path: Path) -> Optional[int]:
|
|||
return duration
|
||||
|
||||
|
||||
def check_for_video_corruption(video_path: Path) -> bool:
|
||||
"""Uses ffprobe to check for specific ways a video file can be corrupt.
|
||||
|
||||
Also attempts to screenshot the video near the end of it's duration, this check if the video is corrupted somewhere
|
||||
in the middle."""
|
||||
video_path = Path(video_path) if not isinstance(video_path, Path) else video_path
|
||||
|
||||
if not video_path.is_file():
|
||||
raise FileNotFoundError(f'{video_path} does not exist!')
|
||||
if not FFPROBE_BIN:
|
||||
raise SystemError('ffprobe is not installed!')
|
||||
|
||||
# Just read the video using FFMPEG, this prints out useful information about the video.
|
||||
cmd = (FFPROBE_BIN, str(video_path))
|
||||
try:
|
||||
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f'FFPROBE failed to check for corruption with stderr: {e.stderr.decode()}')
|
||||
return True # video is corrupt.
|
||||
|
||||
messages = (
|
||||
b'Invalid NAL unit size',
|
||||
b'Error splitting the input into NAL units',
|
||||
b'Invalid data found when processing input',
|
||||
)
|
||||
corrupt = False
|
||||
for error in messages:
|
||||
if error in proc.stderr:
|
||||
logger.warning(f'Possible video corruption ({error.decode()}): {video_path}')
|
||||
corrupt = True
|
||||
return corrupt
|
||||
|
||||
|
||||
async def update_view_counts_and_censored(channel_id: int):
|
||||
"""Update view_count for all Videos in a channel using its info_json file. Also sets FileGroup.censored
|
||||
if Video is no longer available on the Channel."""
|
||||
|
|
|
|||
|
|
@ -40,14 +40,16 @@ def channel_factory(test_session, test_directory):
|
|||
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
|
||||
directory: pathlib.Path = None, tag_name: str = None):
|
||||
name = name or str(uuid4())
|
||||
tag = Tag.get_by_name(tag_name) if tag_name else None
|
||||
tag = Tag.find_by_name(tag_name) if tag_name else None
|
||||
channel = Channel(
|
||||
name=name,
|
||||
url=url or f'https://example.com/{name}',
|
||||
source_id=source_id,
|
||||
tag=tag,
|
||||
tag_id=tag.id if tag else None,
|
||||
)
|
||||
channel.directory = directory or format_videos_destination(name, tag, channel.url)
|
||||
tag_name = tag.name if tag else None
|
||||
channel.directory = directory or format_videos_destination(name, tag_name, channel.url)
|
||||
channel.directory.mkdir(exist_ok=True, parents=True)
|
||||
test_session.add(channel)
|
||||
test_session.flush([channel])
|
||||
|
|
@ -242,12 +244,12 @@ def video_with_search_factory(test_session, test_directory, video_file_factory):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_video_search(test_client):
|
||||
def assert_video_search(test_async_client):
|
||||
"""A fixture which performs a video search request against the API.
|
||||
|
||||
If assert_* params are passed, the response is checked."""
|
||||
|
||||
def _search_videos(
|
||||
async def _search_videos(
|
||||
assert_total: int = None,
|
||||
assert_ids: List[int] = None,
|
||||
assert_paths: List[str] = None,
|
||||
|
|
@ -272,7 +274,7 @@ def assert_video_search(test_client):
|
|||
if channel_id is not None:
|
||||
content['channel_id'] = channel_id
|
||||
|
||||
request, response = test_client.post('/api/videos/search', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/videos/search', content=json.dumps(content))
|
||||
|
||||
assert response.json
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import os.path
|
|||
import pathlib
|
||||
import traceback
|
||||
from abc import ABC
|
||||
from typing import Tuple, List, Dict
|
||||
from typing import Tuple, List, Dict, Optional
|
||||
|
||||
import yt_dlp.utils
|
||||
from sqlalchemy.orm import Session
|
||||
|
|
@ -64,6 +64,7 @@ def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict:
|
|||
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
|
||||
"""Get filename from YoutubeDL. Separated for testing."""
|
||||
dir_name, file_name = os.path.split(ydl.prepare_filename(entry))
|
||||
file_name = escape_file_name(file_name)
|
||||
file_name = trim_file_name(file_name)
|
||||
return f'{dir_name}/{file_name}'
|
||||
|
||||
|
|
@ -241,8 +242,8 @@ class VideoDownloader(Downloader, ABC):
|
|||
|
||||
# Video may have been downloaded previously, get its location for error reporting.
|
||||
with get_db_session() as session:
|
||||
videos = session.query(Video).join(FileGroup).filter_by(url=url).all()
|
||||
location = videos[0].location if videos else None
|
||||
video_ = Video.get_by_url(url, session)
|
||||
location = video_.location if video_ else None
|
||||
|
||||
try:
|
||||
download.info_json = download.info_json or extract_info(url)
|
||||
|
|
@ -257,62 +258,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
if not download.info_json:
|
||||
raise ValueError(f'Cannot download video with no info_json.')
|
||||
|
||||
found_channel = None
|
||||
|
||||
# Look for Channel using channel data first, fallback to uploader data.
|
||||
channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
|
||||
channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
|
||||
channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
|
||||
channel = None
|
||||
if channel_name or channel_source_id or channel_url:
|
||||
# Try to find the channel via info_json from yt-dlp.
|
||||
try:
|
||||
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
|
||||
if channel:
|
||||
found_channel = 'yt_dlp'
|
||||
logger.debug(f'Found a channel with source_id {channel=}')
|
||||
except UnknownChannel:
|
||||
# Can't find a channel, use the no channel directory.
|
||||
pass
|
||||
|
||||
settings = download.settings or dict()
|
||||
destination = settings.get('destination')
|
||||
if not channel and destination:
|
||||
# Destination may find Channel if not already found.
|
||||
try:
|
||||
channel = get_channel(directory=destination, return_dict=False)
|
||||
found_channel = 'download_settings_directory'
|
||||
logger.debug(f'Found a channel that shares the destination directory {channel=}')
|
||||
except UnknownChannel:
|
||||
# Destination must not be a channel.
|
||||
pass
|
||||
|
||||
local_channel_id = settings.get('channel_id')
|
||||
channel_url = settings.get('channel_url')
|
||||
if not channel and (local_channel_id or channel_url):
|
||||
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
|
||||
logger.info(f'Using download.settings to find channel')
|
||||
try:
|
||||
channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
|
||||
logger.debug(f'Found channel with channel url {channel=}')
|
||||
except UnknownChannel:
|
||||
# We do not need a Channel if we have a destination directory.
|
||||
if not destination:
|
||||
raise
|
||||
if channel:
|
||||
found_channel = 'download_settings'
|
||||
|
||||
channel_id = channel.id if channel else None
|
||||
channel_directory = channel.directory if channel else None
|
||||
|
||||
if found_channel == 'yt_dlp':
|
||||
logger.debug(f'Found {channel} using yt_dlp')
|
||||
elif found_channel == 'download_settings':
|
||||
logger.info(f'Found {channel} using Download.settings')
|
||||
elif found_channel == 'download_settings_directory':
|
||||
logger.info(f'Found {channel} using Download.settings directory')
|
||||
else:
|
||||
logger.warning('Could not find channel')
|
||||
channel, channel_directory, channel_id, destination, settings = await self._get_channel(download)
|
||||
|
||||
if destination:
|
||||
# Download to the directory specified in the settings.
|
||||
|
|
@ -480,6 +426,63 @@ class VideoDownloader(Downloader, ABC):
|
|||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
async def _get_channel(download: Download) \
|
||||
-> Tuple[Optional[Channel], Optional[pathlib.Path], Optional[int], Optional[pathlib.Path], dict]:
|
||||
found_channel = None
|
||||
# Look for Channel using channel data first, fallback to uploader data.
|
||||
channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
|
||||
channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
|
||||
channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
|
||||
channel = None
|
||||
if channel_name or channel_source_id or channel_url:
|
||||
# Try to find the channel via info_json from yt-dlp.
|
||||
try:
|
||||
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
|
||||
if channel:
|
||||
found_channel = 'yt_dlp'
|
||||
logger.debug(f'Found a channel with source_id {channel=}')
|
||||
except UnknownChannel:
|
||||
# Can't find a channel, use the no channel directory.
|
||||
pass
|
||||
settings = download.settings or dict()
|
||||
destination = settings.get('destination')
|
||||
if not channel and destination:
|
||||
# Destination may find Channel if not already found.
|
||||
try:
|
||||
channel = get_channel(directory=destination, return_dict=False)
|
||||
found_channel = 'download_settings_directory'
|
||||
logger.debug(f'Found a channel that shares the destination directory {channel=}')
|
||||
except UnknownChannel:
|
||||
# Destination must not be a channel.
|
||||
pass
|
||||
channel_id = settings.get('channel_id')
|
||||
channel_url = settings.get('channel_url')
|
||||
if not channel and (channel_id or channel_url):
|
||||
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
|
||||
logger.info(f'Using download.settings to find channel')
|
||||
try:
|
||||
channel = get_channel(channel_id=channel_id, url=channel_url, return_dict=False)
|
||||
logger.debug(f'Found channel with channel url {channel=}')
|
||||
except UnknownChannel:
|
||||
# We do not need a Channel if we have a destination directory.
|
||||
if not destination:
|
||||
raise
|
||||
if channel:
|
||||
found_channel = 'download_settings'
|
||||
channel_id = channel.id if channel else None
|
||||
channel_directory = channel.directory if channel else None
|
||||
if found_channel == 'yt_dlp':
|
||||
logger.debug(f'Found {channel} using yt_dlp')
|
||||
elif found_channel == 'download_settings':
|
||||
logger.info(f'Found {channel} using Download.settings')
|
||||
elif found_channel == 'download_settings_directory':
|
||||
logger.info(f'Found {channel} using Download.settings directory')
|
||||
else:
|
||||
logger.warning('Could not find channel')
|
||||
|
||||
return channel, channel_directory, channel_id, destination, settings
|
||||
|
||||
@staticmethod
|
||||
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
|
||||
"""Get the full path of a video file from its URL."""
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import dataclasses
|
|||
import html
|
||||
import pathlib
|
||||
import re
|
||||
import traceback
|
||||
from typing import Generator
|
||||
from typing import Tuple, Optional
|
||||
|
||||
|
|
@ -15,7 +16,7 @@ from modules.videos.models import Video
|
|||
from wrolpi import before_startup, dates, flags
|
||||
from wrolpi.captions import extract_captions
|
||||
from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \
|
||||
escape_file_name, get_media_directory
|
||||
escape_file_name, get_media_directory, background_task
|
||||
from wrolpi.dates import Seconds, from_timestamp
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download
|
||||
|
|
@ -379,6 +380,22 @@ def save_channels_config(session: Session = None):
|
|||
channel_import_logger = logger.getChild('channel_import')
|
||||
|
||||
|
||||
async def fetch_channel_source_id(channel_id: int):
|
||||
# If we can download from a channel, we must have its source_id.
|
||||
try:
|
||||
with get_db_session() as session:
|
||||
from modules.videos.channel.lib import get_channel
|
||||
channel = Channel.find_by_id(channel_id)
|
||||
channel.source_id = channel.source_id or get_channel_source_id(channel.url)
|
||||
if not channel.source_id:
|
||||
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
|
||||
else:
|
||||
session.commit()
|
||||
save_channels_config(session)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to get Channel source id of id={channel_id}', exc_info=e)
|
||||
|
||||
|
||||
@before_startup
|
||||
@limit_concurrent(1)
|
||||
def import_channels_config():
|
||||
|
|
@ -387,13 +404,17 @@ def import_channels_config():
|
|||
logger.warning('Skipping import_channels_config for this test')
|
||||
return
|
||||
|
||||
from .channel.lib import get_channel
|
||||
channel_import_logger.info('Importing channels config')
|
||||
try:
|
||||
config = get_channels_config()
|
||||
channels = config.channels
|
||||
channel_directories = [i['directory'] for i in channels]
|
||||
if len(channel_directories) != len(set(channel_directories)):
|
||||
logger.error('Refusing to import channels config because it contains duplicate directories!')
|
||||
return
|
||||
updated_channel_ids = set()
|
||||
|
||||
save_config = False
|
||||
with get_db_session(commit=True) as session:
|
||||
for data in channels:
|
||||
if isinstance(data, str):
|
||||
|
|
@ -405,45 +426,37 @@ def import_channels_config():
|
|||
for option in (i for i in REQUIRED_OPTIONS if i not in data):
|
||||
raise ConfigError(f'Channel "{directory}" is required to have "{option}"')
|
||||
|
||||
channel = session.query(Channel).filter(
|
||||
or_(
|
||||
Channel.directory == str(directory),
|
||||
Channel.directory == str(data['directory']),
|
||||
Channel.directory == str(directory.relative_to(get_media_directory())),
|
||||
),
|
||||
).one_or_none()
|
||||
# Try to find Channel by directory because it is unique.
|
||||
channel = Channel.get_by_path(directory, session)
|
||||
if not channel:
|
||||
# Channel not yet in the DB, add it.
|
||||
channel = Channel(directory=directory)
|
||||
session.add(channel)
|
||||
session.flush([channel, ])
|
||||
# TODO refresh the files in the channel.
|
||||
logger.warning(f'Creating new Channel from config {channel}')
|
||||
|
||||
# Only name and directory are required
|
||||
channel.name = data['name']
|
||||
channel.directory = directory
|
||||
|
||||
# A URL should not be an empty string
|
||||
data['url'] = data['url'] or None
|
||||
try:
|
||||
# Try to find Channel using other attributes before creating new Channel.
|
||||
channel = get_channel(
|
||||
session,
|
||||
source_id=data.get('source_id'),
|
||||
url=data.get('url'),
|
||||
directory=str(directory),
|
||||
return_dict=False,
|
||||
)
|
||||
except UnknownChannel:
|
||||
# Channel not yet in the DB, add it.
|
||||
channel = Channel(directory=directory)
|
||||
session.add(channel)
|
||||
channel.flush()
|
||||
# TODO refresh the files in the channel.
|
||||
logger.warning(f'Creating new Channel from config {channel}')
|
||||
|
||||
# Copy existing channel data, update all values from the config. This is necessary to clear out
|
||||
# values not in the config.
|
||||
full_data = channel.dict()
|
||||
full_data.update(data)
|
||||
channel.update(full_data)
|
||||
updated_channel_ids.add(channel.id)
|
||||
|
||||
if not channel.source_id and channel.url and flags.have_internet.is_set():
|
||||
# If we can download from a channel, we must have its source_id.
|
||||
try:
|
||||
channel.source_id = get_channel_source_id(channel.url)
|
||||
save_config = True
|
||||
if not channel.source_id:
|
||||
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
|
||||
else:
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to get Channel source id of {channel}', exc_info=e)
|
||||
logger.info(f'Fetching channel source id for {channel}')
|
||||
background_task(fetch_channel_source_id(channel.id))
|
||||
|
||||
channel_import_logger.debug(f'Updated {repr(channel.name)}'
|
||||
f' url={channel.url}'
|
||||
|
|
@ -454,17 +467,13 @@ def import_channels_config():
|
|||
with get_db_session(commit=True) as session:
|
||||
# Delete any Channels that were deleted from the config.
|
||||
for channel in session.query(Channel):
|
||||
if str(channel.directory) not in channel_directories:
|
||||
if channel.id not in updated_channel_ids:
|
||||
logger.warning(f'Deleting {channel} because it is not in the config.')
|
||||
channel.delete_with_videos()
|
||||
|
||||
# Create any missing Downloads. Associated Downloads with any necessary Channels.
|
||||
link_channel_and_downloads()
|
||||
|
||||
if save_config:
|
||||
# Information about the channel was fetched, store it.
|
||||
save_channels_config()
|
||||
|
||||
except Exception as e:
|
||||
channel_import_logger.warning('Failed to load channels config!', exc_info=e)
|
||||
if PYTEST:
|
||||
|
|
@ -544,7 +553,8 @@ async def get_statistics():
|
|||
COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE ) AS "have_comments",
|
||||
COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE
|
||||
and fg.censored = false and fg.url is not null) AS "missing_comments",
|
||||
COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments"
|
||||
COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments",
|
||||
COUNT(v.id) FILTER ( WHERE fg.censored = TRUE ) as "censored_videos"
|
||||
FROM
|
||||
video v
|
||||
LEFT JOIN file_group fg on v.file_group_id = fg.id
|
||||
|
|
@ -583,9 +593,11 @@ async def get_statistics():
|
|||
|
||||
curs.execute('''
|
||||
SELECT
|
||||
COUNT(id) AS "channels"
|
||||
COUNT(c.id) AS "channels",
|
||||
COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
|
||||
FROM
|
||||
channel
|
||||
channel c
|
||||
LEFT JOIN public.tag t on t.id = c.tag_id
|
||||
''')
|
||||
channel_stats = dict(curs.fetchone())
|
||||
ret = dict(statistics=dict(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import json
|
||||
import pathlib
|
||||
from typing import Optional, Dict, List
|
||||
|
||||
|
|
@ -7,11 +8,14 @@ from sqlalchemy.orm import relationship, Session, deferred
|
|||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from modules.videos.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi import flags
|
||||
from wrolpi.captions import read_captions
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file, \
|
||||
unique_by_predicate
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download, download_manager, save_downloads_config
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, move
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
|
@ -223,6 +227,7 @@ class Video(ModelHelper, Base):
|
|||
# Can't validate if there is no video file.
|
||||
logger.error(f'Unable to validate video {self.id} without primary file!')
|
||||
|
||||
logger.debug(f'Validating {self}')
|
||||
from .lib import validate_video
|
||||
try:
|
||||
validate_video(self, self.channel.generate_posters if self.channel else False)
|
||||
|
|
@ -237,12 +242,14 @@ class Video(ModelHelper, Base):
|
|||
# self.file_group.d_text is handled in `validate_video`.
|
||||
|
||||
if (comments := self.get_comments()) and len(comments) >= 5:
|
||||
logger.debug(f'{self} has comments')
|
||||
self.have_comments = True
|
||||
|
||||
# If this Video is in a Channel's directory, then it is part of that Channel.
|
||||
session: Session = Session.object_session(self)
|
||||
if session and (channel := Channel.get_by_path(self.video_path.parent, session)):
|
||||
self.channel = channel
|
||||
logger.debug(f'{self} has Channel {channel}')
|
||||
|
||||
@staticmethod
|
||||
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
|
||||
|
|
@ -533,7 +540,7 @@ class Channel(ModelHelper, Base):
|
|||
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
|
||||
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
|
||||
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'))
|
||||
tag = relationship('Tag')
|
||||
tag = relationship('Tag', primaryjoin='Channel.tag_id==Tag.id')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'
|
||||
|
|
@ -565,12 +572,9 @@ class Channel(ModelHelper, Base):
|
|||
channel_id = self.id
|
||||
data = data.copy()
|
||||
|
||||
# Remove unused values.
|
||||
data.pop('move_to_tag_directory', None)
|
||||
|
||||
# URL should not be empty string.
|
||||
if 'url' in data:
|
||||
data['url'] = data['url'] or None
|
||||
url = data.pop('url')
|
||||
self.url = url or None
|
||||
|
||||
session: Session = Session.object_session(self)
|
||||
|
||||
|
|
@ -664,23 +668,19 @@ class Channel(ModelHelper, Base):
|
|||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT
|
||||
SUM(size),
|
||||
MAX(size),
|
||||
COUNT(video.id),
|
||||
SUM(fg.length)
|
||||
SUM(size) AS "size",
|
||||
MAX(size) AS "largest_video",
|
||||
COUNT(video.id) AS "video_count",
|
||||
SUM(fg.length) AS "length",
|
||||
-- Videos may use multiple tags.
|
||||
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
|
||||
FROM video
|
||||
LEFT JOIN file_group fg on fg.id = video.file_group_id
|
||||
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
|
||||
WHERE channel_id = %(id)s
|
||||
'''
|
||||
curs.execute(stmt, dict(id=self.id))
|
||||
size, largest_video, video_count, length = curs.fetchone()
|
||||
statistics = dict(
|
||||
video_count=video_count,
|
||||
size=size,
|
||||
largest_video=largest_video,
|
||||
length=length,
|
||||
)
|
||||
return statistics
|
||||
return dict(curs.fetchone())
|
||||
|
||||
async def refresh_files(self, send_events: bool = True):
|
||||
"""Refresh all files within this Channel's directory. Mark this channel as refreshed."""
|
||||
|
|
@ -772,39 +772,72 @@ class Channel(ModelHelper, Base):
|
|||
channel = session.query(Channel).filter_by(url=url).one_or_none()
|
||||
return channel
|
||||
|
||||
async def move(self, directory: pathlib.Path, session: Session):
|
||||
def format_directory(self, tag_name: str) -> pathlib.Path:
|
||||
from modules.videos.lib import format_videos_destination
|
||||
return format_videos_destination(self.name, tag_name, self.url)
|
||||
|
||||
async def move_channel(self, directory: pathlib.Path, session: Session, send_events: bool = False):
|
||||
"""Move the files of this Channel into a new directory."""
|
||||
if not directory.is_dir():
|
||||
raise FileNotFoundError(f'Destination directory does not exist: {directory}')
|
||||
if self.directory and not next(self.directory.iterdir(), None):
|
||||
raise FileExistsError(f'Refusing to move Channel when it has no files.')
|
||||
|
||||
old_directory = self.directory
|
||||
self.directory = directory
|
||||
session.flush([self, ])
|
||||
|
||||
# Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
|
||||
# directory.
|
||||
downloads = list(self.downloads)
|
||||
downloads.extend(list(
|
||||
session.query(Download).filter(Download.settings.contains({'destination': str(old_directory)}))
|
||||
))
|
||||
for download in downloads:
|
||||
# Get new copy of `settings` to avoid bug where it wouldn't update.
|
||||
settings = download.settings.copy() if download.settings else dict()
|
||||
if settings.get('destination'):
|
||||
download.settings = {**settings, 'destination': str(directory)}
|
||||
def change_download_destinations(from_directory: pathlib.Path, to_directory: pathlib.Path):
|
||||
downloads = list(self.downloads)
|
||||
downloads.extend(Download.get_all_by_destination(from_directory))
|
||||
downloads = unique_by_predicate(downloads, lambda i: i.id)
|
||||
for download in downloads:
|
||||
# Get new copy of `settings` to avoid bug where it wouldn't update.
|
||||
settings = download.settings.copy() if download.settings else dict()
|
||||
if settings.get('destination'):
|
||||
logger.debug(f'Updating destination of {download}')
|
||||
download.settings = {**settings, 'destination': str(to_directory)}
|
||||
session.flush(downloads)
|
||||
|
||||
session.commit()
|
||||
# Only one tag can be moved at a time.
|
||||
with flags.refreshing:
|
||||
# Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
|
||||
# directory.
|
||||
change_download_destinations(old_directory, directory)
|
||||
|
||||
# Save configs before move, this is because move imports configs.
|
||||
from modules.videos.lib import save_channels_config
|
||||
await save_downloads_config(session)
|
||||
save_channels_config(session)
|
||||
save_tags_config(session)
|
||||
session.commit()
|
||||
|
||||
# Move the contents of the Channel directory into the destination directory.
|
||||
await move(directory, *list(old_directory.iterdir()))
|
||||
# Save configs before move, this is because move imports configs.
|
||||
from modules.videos.lib import save_channels_config
|
||||
await save_downloads_config(session)
|
||||
save_channels_config(session)
|
||||
save_tags_config(session)
|
||||
|
||||
if old_directory.exists() and not next(old_directory.iterdir(), None):
|
||||
old_directory.rmdir()
|
||||
# Move the contents of the Channel directory into the destination directory.
|
||||
logger.info(f'Moving {self} from {repr(str(old_directory))}')
|
||||
|
||||
try:
|
||||
if not old_directory.exists():
|
||||
# Old directory does not exist, maintainer must have moved the Channel manually.
|
||||
if PYTEST:
|
||||
await refresh_files([old_directory, directory])
|
||||
else:
|
||||
background_task(refresh_files([old_directory, directory]))
|
||||
if send_events:
|
||||
Events.send_file_move_completed(f'Channel {repr(self.name)} was moved, but Tags were lost')
|
||||
else:
|
||||
# `move` also uses `flags.refreshing`
|
||||
await move(directory, *list(old_directory.iterdir()))
|
||||
if send_events:
|
||||
Events.send_file_move_completed(f'Channel {repr(self.name)} was moved')
|
||||
except Exception as e:
|
||||
logger.error(f'Channel move failed! Reverting changes...', exc_info=e)
|
||||
# Downloads must be moved back to the old directory.
|
||||
change_download_destinations(directory, old_directory)
|
||||
self.directory = old_directory
|
||||
self.flush(session)
|
||||
if send_events:
|
||||
Events.send_file_move_failed(f'Moving Channel {self.name} has failed')
|
||||
raise
|
||||
finally:
|
||||
session.commit()
|
||||
if old_directory.exists() and not next(iter(old_directory.iterdir()), None):
|
||||
# Old directory is empty, delete it.
|
||||
old_directory.rmdir()
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ class ChannelTagRequest:
|
|||
|
||||
@dataclass
|
||||
class ChannelTagInfoRequest:
|
||||
channel_id: Optional[int] = None
|
||||
tag_name: Optional[str] = None
|
||||
|
||||
|
||||
|
|
@ -205,6 +206,7 @@ class VideoStatistics:
|
|||
@dataclass
|
||||
class ChannelStatistics:
|
||||
channels: int
|
||||
tagged_channels: int
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -222,3 +224,8 @@ class CensoredVideoRequest:
|
|||
@dataclass
|
||||
class CensoredVideoResponse:
|
||||
videos: List[VideoWithChannel]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelSearchRequest:
|
||||
tag_names: List[str] = field(default_factory=list)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import pytest
|
|||
|
||||
from modules.videos.models import Video, Channel
|
||||
from modules.videos.video import lib as video_lib
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.files.lib import refresh_files
|
||||
|
|
@ -160,14 +161,15 @@ def test_api_download(test_session, test_client, test_directory):
|
|||
assert not download.settings.get('tag_names')
|
||||
|
||||
|
||||
def test_api_download_video_tags(test_session, test_client, tag_factory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_download_video_tags(test_session, test_async_client, tag_factory):
|
||||
"""A user can request Tags for a video being downloaded."""
|
||||
tag1 = tag_factory()
|
||||
tag2 = tag_factory()
|
||||
tag1 = await tag_factory()
|
||||
tag2 = await tag_factory()
|
||||
|
||||
content = dict(urls=['https://example.com/video1', ], downloader='video',
|
||||
settings={'tag_names': [tag1.name, tag2.name]})
|
||||
request, response = test_client.post('/api/download', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/download', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
download = test_session.query(Download).one()
|
||||
|
|
@ -176,7 +178,9 @@ def test_api_download_video_tags(test_session, test_client, tag_factory):
|
|||
assert download.settings['tag_names'] == [tag1.name, tag2.name]
|
||||
|
||||
|
||||
def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, assert_video_search):
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory,
|
||||
assert_video_search):
|
||||
"""Test that videos can be searched and that their order is by their textsearch rank."""
|
||||
# These captions have repeated letters, so they will be higher in the ranking.
|
||||
videos = [
|
||||
|
|
@ -193,37 +197,38 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
|
|||
# Repeated runs should return the same result
|
||||
for _ in range(2):
|
||||
# Only videos with a b are returned, ordered by the amount of b's
|
||||
assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
|
||||
await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
|
||||
|
||||
# Only two captions have e
|
||||
assert_video_search(search_str='e', assert_ids=[4, 1])
|
||||
await assert_video_search(search_str='e', assert_ids=[4, 1])
|
||||
|
||||
# Only 2 contains 2
|
||||
assert_video_search(search_str='2', assert_ids=[2, ])
|
||||
await assert_video_search(search_str='2', assert_ids=[2, ])
|
||||
|
||||
# Only two captions have d
|
||||
assert_video_search(search_str='d', assert_ids=[1, 2])
|
||||
await assert_video_search(search_str='d', assert_ids=[1, 2])
|
||||
|
||||
# 5 can be gotten by it's title
|
||||
assert_video_search(search_str='5', assert_ids=[5, ])
|
||||
await assert_video_search(search_str='5', assert_ids=[5, ])
|
||||
|
||||
# only video 1 has e and d
|
||||
assert_video_search(search_str='e d', assert_ids=[1, ])
|
||||
await assert_video_search(search_str='e d', assert_ids=[1, ])
|
||||
|
||||
# video 1 and 4 have b and e, but 1 has more
|
||||
assert_video_search(search_str='b e', assert_ids=[1, 4])
|
||||
await assert_video_search(search_str='b e', assert_ids=[1, 4])
|
||||
|
||||
# Check totals are correct even with a limit
|
||||
assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
|
||||
await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
|
||||
|
||||
|
||||
def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
|
||||
"""Search the Video table. This does not need to use a join with the File table."""
|
||||
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
|
||||
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2',
|
||||
channel_id=simple_channel.id)
|
||||
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
|
||||
vid1.add_tag(tag1.id)
|
||||
|
||||
|
|
@ -237,22 +242,68 @@ def test_search_videos(test_session, video_factory, assert_video_search, simple_
|
|||
assert test_session.query(Video).count() == 3
|
||||
assert test_session.query(FileGroup).count() == 3
|
||||
|
||||
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
|
||||
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2,
|
||||
offset=1)
|
||||
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
|
||||
channel_id=simple_channel.id)
|
||||
assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
|
||||
assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
|
||||
assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
|
||||
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
|
||||
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
|
||||
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
|
||||
await assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2,
|
||||
offset=1)
|
||||
await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
|
||||
channel_id=simple_channel.id)
|
||||
await assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
|
||||
await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
|
||||
await assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
|
||||
# Only vid2 has both tags.
|
||||
assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
|
||||
await assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
|
||||
|
||||
# No results, no total is returned from the DB.
|
||||
assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
|
||||
await assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
|
||||
|
||||
# Check all order_by.
|
||||
for order_by in video_lib.VIDEO_ORDERS:
|
||||
assert_video_search(order_by=order_by)
|
||||
await assert_video_search(order_by=order_by)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_video(test_async_client, test_session, video_factory, test_download_manager):
|
||||
"""When a Video record is deleted, the FileGroup should be deleted. The URL is added to the
|
||||
global skip list so the video is not downloaded again."""
|
||||
video = video_factory(with_video_file=True, with_poster_ext='png')
|
||||
video.file_group.url = 'https://example.com/video/1'
|
||||
files = video.file_group.my_paths()
|
||||
test_session.commit()
|
||||
|
||||
request, response = await test_async_client.delete(f'/api/videos/video/{video.id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
|
||||
assert test_session.query(FileGroup).count() == 0, 'Video FileGroup was not deleted.'
|
||||
assert not any(i.exists() for i in files), 'All files should have been deleted.'
|
||||
assert test_download_manager.is_skipped('https://example.com/video/1'), 'Video should not be downloaded again.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_videos_description(test_async_client, test_session, test_directory, channel_factory, tag_factory):
|
||||
channel = channel_factory(name='Channel Name')
|
||||
tag = await tag_factory()
|
||||
test_session.commit()
|
||||
|
||||
body = dict()
|
||||
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['videos_destination'] == 'videos'
|
||||
|
||||
body = dict(channel_id=channel.id)
|
||||
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['videos_destination'] == 'videos/Channel Name'
|
||||
|
||||
body = dict(channel_id=channel.id, tag_name=tag.name)
|
||||
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['videos_destination'] == 'videos/one/Channel Name'
|
||||
|
||||
get_wrolpi_config().videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
|
||||
body = dict(channel_id=channel.id, tag_name=tag.name)
|
||||
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'
|
||||
|
|
|
|||
|
|
@ -135,10 +135,12 @@ def test_generate_video_poster(video_file):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config):
|
||||
async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config,
|
||||
tag_factory):
|
||||
"""Importing the Channels' config should create any missing download records"""
|
||||
channel1 = channel_factory(source_id='foo', url='https://example.com/channel1')
|
||||
channel2 = channel_factory(source_id='bar', url='https://example.com/channel2')
|
||||
tag = await tag_factory()
|
||||
assert len(test_session.query(Channel).all()) == 2
|
||||
test_session.commit()
|
||||
|
||||
|
|
@ -228,8 +230,19 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
|
|||
assert {i.url for i in channel1.downloads} == {'https://example.com/channel1'}
|
||||
assert {i.url for i in channel2.downloads} == {'https://example.com/channel2', 'https://example.org'}
|
||||
|
||||
# A Channel with a `tag_name` is tagged.
|
||||
update_channel_config(get_channels_config(),
|
||||
channel1.source_id,
|
||||
{'tag_name': tag.name})
|
||||
import_channels_config()
|
||||
channel1, channel2 = test_session.query(Channel).order_by(Channel.url).all()
|
||||
assert channel1.tag == tag and channel1.tag_id == tag.id
|
||||
assert channel2.tag is None and channel2.tag_id is None
|
||||
|
||||
def test_import_channel_delete_missing_channels(test_session, channel_factory, test_channels_config):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_channel_delete_missing_channels(
|
||||
test_async_client, test_session, channel_factory, test_channels_config):
|
||||
"""The Channel import function deletes any Channels that are not in the config."""
|
||||
# Create a DB and config with two Channels.
|
||||
channel1 = channel_factory(source_id='foo')
|
||||
|
|
@ -268,32 +281,6 @@ def test_import_channel_delete_missing_channels(test_session, channel_factory, t
|
|||
assert str(channel2.directory) not in test_channels_config.read_text()
|
||||
|
||||
|
||||
def test_check_for_video_corruption(video_file, test_directory):
|
||||
# The test video is not corrupt.
|
||||
assert common.check_for_video_corruption(video_file) is False
|
||||
|
||||
# An empty file is corrupt.
|
||||
empty_file = test_directory / 'empty file.mp4'
|
||||
empty_file.touch()
|
||||
assert common.check_for_video_corruption(empty_file) is True
|
||||
|
||||
# A video file must be complete.
|
||||
truncated_video = test_directory / 'truncated_video.mp4'
|
||||
with truncated_video.open('wb') as fh:
|
||||
fh.write(video_file.read_bytes()[:10000])
|
||||
assert common.check_for_video_corruption(truncated_video) is True
|
||||
|
||||
# Check for specific ffprobe errors.
|
||||
with mock.patch('modules.videos.common.subprocess') as mock_subprocess:
|
||||
# `video_file` is ignored for these calls.
|
||||
mock_subprocess.run().stderr = b'Something\nInvalid NAL unit size'
|
||||
assert common.check_for_video_corruption(video_file) is True
|
||||
mock_subprocess.run().stderr = b'Something\nError splitting the input into NAL units'
|
||||
assert common.check_for_video_corruption(video_file) is True
|
||||
mock_subprocess.run().stderr = b'Some stderr is fine'
|
||||
assert common.check_for_video_corruption(video_file) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ffprobe_json(test_async_client, video_file, corrupted_video_file):
|
||||
content = await common.ffprobe_json(video_file)
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
|
|||
mock_video_extract_info, mock_video_process_runner, tag_factory):
|
||||
"""A Video is tagged when Download record requires it."""
|
||||
video_path = video_file_factory()
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
mock_video_extract_info.return_value = copy(example_video_json)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ async def test_channel_move(test_async_client, test_session, test_directory, cha
|
|||
foo.mkdir(parents=True)
|
||||
|
||||
# Move the Channel.
|
||||
await channel.move(foo, test_session)
|
||||
await channel.move_channel(foo, test_session)
|
||||
|
||||
assert channel.directory == foo
|
||||
assert channel.directory != (test_directory / 'New Channel Directory')
|
||||
|
|
|
|||
|
|
@ -29,27 +29,11 @@ def test_delete_video_no_channel(test_session, simple_video):
|
|||
assert test_session.query(Video).count() == 0
|
||||
|
||||
|
||||
def test_delete_video(test_session, video_factory):
|
||||
"""When a Video record is deleted, all referenced file records should be deleted."""
|
||||
video = video_factory(with_video_file=True, with_poster_ext='png')
|
||||
test_session.commit()
|
||||
|
||||
assert video.video_path.is_file(), 'Video file was not created.'
|
||||
assert video.poster_path.is_file(), 'Video poster was not created.'
|
||||
assert test_session.query(Video).count() == 1, 'Video was not created.'
|
||||
assert test_session.query(FileGroup).count() == 1, 'Video file and poster file were not created.'
|
||||
|
||||
video.delete()
|
||||
test_session.commit()
|
||||
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
|
||||
assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory):
|
||||
"""You cannot delete a video with a tag."""
|
||||
video = video_factory(with_video_file=True, with_poster_ext='png')
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
video.add_tag(tag.name)
|
||||
test_session.commit()
|
||||
|
||||
|
|
@ -63,7 +47,7 @@ async def test_delete_video_with_tag(test_async_client, test_session, video_fact
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_channel_refresh(test_session, test_directory, channel_factory, video_factory):
|
||||
async def test_video_channel_refresh(test_async_client, test_session, test_directory, channel_factory, video_factory):
|
||||
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
|
||||
# Create a video file in this channel's directory.
|
||||
channel = channel_factory()
|
||||
|
|
@ -101,7 +85,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
|
|||
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
|
||||
entry = channel.info_json['entries'][0]
|
||||
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
vid1 = video_factory(
|
||||
channel_id=channel.id,
|
||||
title=f'{channel.name}_20000101_ABC123456_The video title',
|
||||
|
|
@ -150,7 +134,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_renamed_video(test_session, channel_factory, video_factory, tag_factory):
|
||||
async def test_delete_renamed_video(test_async_client, test_session, channel_factory, video_factory, tag_factory):
|
||||
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
|
||||
channel = channel_factory(name='Channel Name')
|
||||
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'
|
||||
|
|
@ -158,7 +142,7 @@ async def test_delete_renamed_video(test_session, channel_factory, video_factory
|
|||
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]}
|
||||
entry = channel.info_json['entries'][0]
|
||||
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
vid1 = video_factory(
|
||||
channel_id=channel.id,
|
||||
title=f'{channel.name}_20000101_ABC123456_The video title',
|
||||
|
|
|
|||
|
|
@ -164,8 +164,11 @@ def delete_videos(*video_ids: int, session: Session = None):
|
|||
raise UnknownVideo('Could not find videos to delete')
|
||||
|
||||
logger.warning(f'Deleting {len(videos)} videos')
|
||||
# Get all URLs, skip them once (so config isn't saved multiple times).
|
||||
urls = list(filter(None, [i.file_group.url for i in videos]))
|
||||
download_manager.add_to_skip_list(*urls)
|
||||
for video in videos:
|
||||
video.delete(add_to_skip_list=True)
|
||||
video.delete()
|
||||
session.commit()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from modules.videos.models import Video
|
|||
from wrolpi.dates import now
|
||||
|
||||
|
||||
def test_get_video_prev_next(test_session, channel_factory, video_factory):
|
||||
def test_get_video_prev_next(test_async_client, test_session, channel_factory, video_factory):
|
||||
"""
|
||||
Test that the previous and next videos will be retrieved when fetching a video.
|
||||
"""
|
||||
|
|
@ -69,7 +69,7 @@ def test_get_video_prev_next(test_session, channel_factory, video_factory):
|
|||
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
|
||||
|
||||
|
||||
def test_get_video_prev_next_no_upload_date(test_session, video_factory, channel_factory):
|
||||
def test_get_video_prev_next_no_upload_date(test_async_client, test_session, video_factory, channel_factory):
|
||||
"""
|
||||
Previous and Next Videos should be those next to a Video file when videos do not have upload dates.
|
||||
"""
|
||||
|
|
@ -153,7 +153,7 @@ async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_m
|
|||
test_download_manager, tag_factory):
|
||||
"""Many methods are blocked when WROL Mode is enabled."""
|
||||
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
|
||||
await wrol_mode_fixture(True)
|
||||
|
||||
|
|
|
|||
|
|
@ -100,14 +100,14 @@ def zim_factory(test_session, zim_path_factory, test_directory):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_zim_search(test_client, test_directory):
|
||||
def assert_zim_search(test_async_client, test_directory):
|
||||
from http import HTTPStatus
|
||||
|
||||
def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
|
||||
expected_status_code: int = HTTPStatus.OK):
|
||||
async def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
|
||||
expected_status_code: int = HTTPStatus.OK):
|
||||
content = dict(search_str=search_str, tag_names=tag_names)
|
||||
assert isinstance(zim_id, int), 'This fixture only supports Zim ids of type integer'
|
||||
request, response = test_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
|
||||
request, response = await test_async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
|
||||
assert response.status_code == expected_status_code
|
||||
|
||||
result = response.json['zim']
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from modules.zim.kiwix import KIWIX_CATALOG
|
|||
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
|
||||
register_refresh_cleanup, background_task, get_wrolpi_config
|
||||
register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate
|
||||
from wrolpi.db import get_db_session, optional_session, get_db_curs
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.files.lib import refresh_files, split_file_name_words
|
||||
|
|
@ -124,14 +124,11 @@ def get_entries_tags(paths: List[str], zim_id: int):
|
|||
|
||||
def get_unique_paths(*paths: str) -> List[str]:
|
||||
"""Return a new list which contains unique Zim paths."""
|
||||
new_paths = list()
|
||||
simple_paths = list()
|
||||
for path in paths:
|
||||
simple_path = path.replace('-', '').replace('_', '').replace('/', '').lower()
|
||||
if simple_path not in simple_paths:
|
||||
new_paths.append(path)
|
||||
simple_paths.append(simple_path)
|
||||
return new_paths
|
||||
unique_paths = unique_by_predicate(
|
||||
paths,
|
||||
lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower()
|
||||
)
|
||||
return list(unique_paths)
|
||||
|
||||
|
||||
@optional_session
|
||||
|
|
|
|||
|
|
@ -8,16 +8,17 @@ from modules.zim.models import Zim
|
|||
from wrolpi.downloader import Download
|
||||
|
||||
|
||||
def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim_search):
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_search(test_async_client, test_session, test_zim, tag_factory, assert_zim_search):
|
||||
"""Entries can be tagged and retrieved using their containing Zim."""
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
test_zim.tag_entry(tag1.name, 'one')
|
||||
test_zim.tag_entry(tag2.name, 'one')
|
||||
test_zim.tag_entry(tag1.name, 'two')
|
||||
test_session.commit()
|
||||
|
||||
# Test searching with only `search_str`.
|
||||
assert_zim_search('one', 1, {
|
||||
await assert_zim_search('one', 1, {
|
||||
'path': test_zim.path,
|
||||
'search': [
|
||||
# Entry "One" contains a 'one' in the title, and the text.
|
||||
|
|
@ -27,13 +28,13 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
|
|||
],
|
||||
'estimate': 2,
|
||||
})
|
||||
assert_zim_search('two', 1, {
|
||||
await assert_zim_search('two', 1, {
|
||||
'path': test_zim.path,
|
||||
'search': [{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
|
||||
{'path': 'home', 'headline': 'testing.\nOne\n<b>Two</b>', 'rank': 0.06079271}],
|
||||
'estimate': 2,
|
||||
})
|
||||
assert_zim_search('item', 1, {
|
||||
await assert_zim_search('item', 1, {
|
||||
'path': test_zim.path,
|
||||
'search': [{'path': 'one', 'headline': 'first <b>item</b>', 'rank': 0.06079271},
|
||||
{'path': 'two', 'headline': 'second <b>item</b>', 'rank': 0.06079271}],
|
||||
|
|
@ -41,35 +42,35 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
|
|||
})
|
||||
|
||||
# Test searching with `search_str` and `tag_names`.
|
||||
assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={
|
||||
await assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={
|
||||
'search': [
|
||||
# Only entry "One" is tagged with Tag 2.
|
||||
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
|
||||
],
|
||||
'estimate': 1,
|
||||
})
|
||||
assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={
|
||||
await assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={
|
||||
'search': [
|
||||
# Only "One" is tagged with Tag 1 and contains "one".
|
||||
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
|
||||
],
|
||||
'estimate': 1,
|
||||
})
|
||||
assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={
|
||||
await assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={
|
||||
'search': [
|
||||
# Only "Two" was tagged with Tag 1 and contains "two".
|
||||
{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
|
||||
],
|
||||
'estimate': 1,
|
||||
})
|
||||
assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={
|
||||
await assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={
|
||||
# Two was not tagged with Tag 1.
|
||||
'search': [],
|
||||
'estimate': 0,
|
||||
})
|
||||
|
||||
# Test search with only `tag_names`.
|
||||
assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={
|
||||
await assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={
|
||||
'search': [
|
||||
# Return all entries that are tagged with the Tags.
|
||||
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.0, 'tag_names': ['one', 'two']},
|
||||
|
|
@ -78,9 +79,10 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
|
|||
})
|
||||
|
||||
|
||||
def test_entries_by_tag(test_client, test_session, test_zim, tag_factory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_factory):
|
||||
"""Entries can be tagged and retrieved using their containing Zim."""
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
test_zim.tag_entry(tag1.name, 'one')
|
||||
test_zim.tag_entry(tag2.name, 'one')
|
||||
test_zim.tag_entry(tag1.name, 'two')
|
||||
|
|
@ -96,36 +98,38 @@ def test_entries_by_tag(test_client, test_session, test_zim, tag_factory):
|
|||
assert [(i.path, i.title) for i in result] == [('one', 'One'), ]
|
||||
|
||||
|
||||
def test_zim_crud(test_client, test_session, test_directory, test_zim):
|
||||
request, response = test_client.get('/api/zim')
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_crud(test_async_client, test_session, test_directory, test_zim):
|
||||
request, response = await test_async_client.get('/api/zim')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['zims']) == 1
|
||||
assert response.json['zims'][0]['path'] == test_zim.path.name
|
||||
|
||||
request, response = test_client.delete('/api/zim/1')
|
||||
request, response = await test_async_client.delete('/api/zim/1')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert not test_zim.path.exists()
|
||||
assert test_session.query(Zim).count() == 0
|
||||
|
||||
|
||||
def test_zim_tag_and_untag(test_client, test_session, test_zim, tag_factory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_tag_and_untag(test_async_client, test_session, test_zim, tag_factory):
|
||||
"""Zim entries can be Tagged and Untagged."""
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
test_session.commit()
|
||||
|
||||
# Entries can be tagged in the API.
|
||||
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
|
||||
request, response = test_client.post('/api/zim/tag', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/zim/tag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
|
||||
# Entries can be untagged in the API.
|
||||
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
|
||||
request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Cannot untag twice.
|
||||
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
|
||||
request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -51,14 +51,14 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory):
|
||||
tag1 = tag_factory()
|
||||
tag1 = await tag_factory()
|
||||
with pytest.raises(UnknownZimEntry):
|
||||
test_zim.tag_entry(tag1.name, 'path does not exist')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory):
|
||||
tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
|
||||
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
||||
test_zim.tag_entry(tag1.name, 'one')
|
||||
test_zim.tag_entry(tag2.name, 'one')
|
||||
test_zim.tag_entry(tag1.name, 'two')
|
||||
|
|
@ -70,7 +70,8 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory):
|
||||
tze = zim_factory('zim1').tag_entry(tag_factory().name, 'one')
|
||||
tag = await tag_factory()
|
||||
tze = zim_factory('zim1').tag_entry(tag.name, 'one')
|
||||
# Test TagZimEntry.__repr__
|
||||
assert repr(tze) == "<TagZimEntry tag='one' zim=zim1 zim_entry=one>"
|
||||
|
||||
|
|
@ -78,7 +79,7 @@ async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory,
|
|||
@pytest.mark.asyncio
|
||||
async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory):
|
||||
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
|
||||
# Tag Zim1 with both tags, but Zim2 shares one of the tags and is returned when no Zim is passed.
|
||||
zim1.tag_entry(tag1.name, 'one')
|
||||
|
|
@ -124,13 +125,14 @@ def test_get_unique_paths():
|
|||
'A/Thomas_Harvey_Johnston']
|
||||
|
||||
|
||||
def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
|
||||
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
|
||||
config = tags.get_tags_config()
|
||||
assert not config.tag_zims
|
||||
|
||||
# Tag three Zim entries.
|
||||
tag1, tag2 = tag_factory('Tag1'), tag_factory('Tag2')
|
||||
tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2')
|
||||
test_zim.tag_entry(tag1.name, 'one')
|
||||
test_zim.tag_entry(tag2.name, 'one')
|
||||
test_zim.tag_entry(tag1.name, 'two')
|
||||
|
|
@ -243,12 +245,13 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te
|
|||
assert not flags.outdated_zims.is_set()
|
||||
|
||||
|
||||
def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
|
||||
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
|
||||
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
|
||||
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
|
||||
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
|
||||
tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
|
||||
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
||||
# Tag outdated Zim entries.
|
||||
zim1.tag_entry(tag1.name, 'home')
|
||||
zim1.tag_entry(tag2.name, 'one')
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import socket
|
|||
import string
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from asyncio import Task
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -160,6 +161,7 @@ __all__ = [
|
|||
'ConfigFile',
|
||||
'DownloadFileInfo',
|
||||
'DownloadFileInfoLink',
|
||||
'LOGGING_CONFIG',
|
||||
'ModelHelper',
|
||||
'WROLPI_CONFIG',
|
||||
'aiohttp_get',
|
||||
|
|
@ -180,7 +182,6 @@ __all__ = [
|
|||
'compile_tsvector',
|
||||
'cum_timer',
|
||||
'date_range',
|
||||
'LOGGING_CONFIG',
|
||||
'disable_wrol_mode',
|
||||
'download_file',
|
||||
'enable_wrol_mode',
|
||||
|
|
@ -227,6 +228,7 @@ __all__ = [
|
|||
'truncate_generator_bytes',
|
||||
'truncate_object_bytes',
|
||||
'tsvector',
|
||||
'unique_by_predicate',
|
||||
'url_strip_host',
|
||||
'walk',
|
||||
'wrol_mode_check',
|
||||
|
|
@ -252,10 +254,6 @@ class ModelHelper:
|
|||
def get_by_path(path, session) -> Base:
|
||||
raise NotImplementedError('This model has not defined this method.')
|
||||
|
||||
@staticmethod
|
||||
def find_by_paths(paths, session) -> List:
|
||||
raise NotImplementedError('This model has not defined this method.')
|
||||
|
||||
@staticmethod
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
|
||||
"""Attempts to get a model instance by its id. Returns None if no instance can be found."""
|
||||
|
|
@ -274,9 +272,10 @@ class ModelHelper:
|
|||
def do_model(file_group, session: Session):
|
||||
raise NotImplementedError('This model has not defined this method.')
|
||||
|
||||
def flush(self):
|
||||
def flush(self, session: Session = None):
|
||||
"""A convenience method which flushes this record using its DB Session."""
|
||||
if session := Session.object_session(self):
|
||||
session = session or Session.object_session(self)
|
||||
if session:
|
||||
session.flush([self])
|
||||
else:
|
||||
raise RuntimeError(f'{self} is not in a session!')
|
||||
|
|
@ -746,6 +745,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
|
|||
synchronous_after = after
|
||||
|
||||
async def after(*a, **kw):
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
traceback.print_stack()
|
||||
logger.debug(f'run_after synchronous_after called for {after}')
|
||||
return synchronous_after(*a, **kw)
|
||||
|
||||
def wrapper(func: callable):
|
||||
|
|
@ -756,6 +758,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
|
|||
@wraps(func)
|
||||
async def wrapped(*a, **kw):
|
||||
results = await func(*a, **kw)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
traceback.print_stack()
|
||||
logger.debug(f'run_after async called for {func}')
|
||||
coro = after(*args, **kwargs)
|
||||
asyncio.ensure_future(coro)
|
||||
return results
|
||||
|
|
@ -763,6 +768,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
|
|||
@wraps(func)
|
||||
def wrapped(*a, **kw):
|
||||
results = func(*a, **kw)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
traceback.print_stack()
|
||||
logger.debug(f'run_after sync called for {func}')
|
||||
coro = after(*args, **kwargs)
|
||||
asyncio.ensure_future(coro)
|
||||
return results
|
||||
|
|
@ -1741,6 +1749,35 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
|
|||
return headlines
|
||||
|
||||
|
||||
async def search_other_estimates(tag_names: List[str]) -> dict:
|
||||
"""Estimate other things that are Tagged."""
|
||||
from wrolpi.db import get_db_curs
|
||||
|
||||
if not tag_names:
|
||||
return dict(
|
||||
channel_count=0,
|
||||
)
|
||||
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT
|
||||
COUNT(c.id)
|
||||
FROM channel c
|
||||
LEFT OUTER JOIN public.tag t on t.id = c.tag_id
|
||||
WHERE t.name = %(tag_name)s
|
||||
'''
|
||||
# TODO handle multiple tags
|
||||
params = dict(tag_name=tag_names[0])
|
||||
curs.execute(stmt, params)
|
||||
|
||||
channel_count = curs.fetchone()[0]
|
||||
|
||||
others = dict(
|
||||
channel_count=channel_count,
|
||||
)
|
||||
return others
|
||||
|
||||
|
||||
def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False):
|
||||
"""Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful.
|
||||
The original file will be restored if any of these steps fail."""
|
||||
|
|
@ -1891,3 +1928,29 @@ def is_valid_hex_color(hex_color: str) -> bool:
|
|||
def is_hardlinked(path: pathlib.Path) -> bool:
|
||||
"""Returns True only if `path` has more than one link."""
|
||||
return path.stat().st_nlink > 1
|
||||
|
||||
|
||||
def _default_predicate(i):
|
||||
return i
|
||||
|
||||
|
||||
def unique_by_predicate(
|
||||
iterable: list | tuple | set,
|
||||
predicate: callable = None,
|
||||
) -> list | tuple | set:
|
||||
"""Finds only unique items in the provided iterable based on the predicate function. Order is preserved.
|
||||
|
||||
`predicate` defaults to the items themselves.
|
||||
|
||||
>>> unique_by_predicate([['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach']], lambda i: len(i))
|
||||
# [ 'apple', 'banana', 'pear' ]
|
||||
>>> unique_by_predicate([Path('foo.txt'), Path('foo.mp4'), Path('bar.txt')], lambda i: i.stem)
|
||||
# [ Path('foo.txt'), Path('bar.txt') ]
|
||||
>>> unique_by_predicate([1, 1, 2, 3, 3, 3, 3, 4], None)
|
||||
# [ 1, 2, 3, 4 ]
|
||||
"""
|
||||
predicate = predicate or _default_predicate # Return an iterable based off the object itself by default.
|
||||
seen = set()
|
||||
unique_items = [i for i in iterable if not ((key := predicate(i)) in seen or seen.add(key))]
|
||||
# Return the same type as the input iterable
|
||||
return iterable.__class__(unique_items) # For other iterable types
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import zipfile
|
|||
from abc import ABC
|
||||
from datetime import datetime
|
||||
from itertools import zip_longest
|
||||
from typing import List, Callable, Dict, Sequence, Union, Coroutine
|
||||
from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional
|
||||
from typing import Tuple, Set
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock, AsyncMock
|
||||
|
|
@ -175,7 +175,7 @@ async def test_async_client(test_directory) -> SanicASGITestClient:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def test_download_manager_config(test_async_client, test_directory):
|
||||
def test_download_manager_config(test_async_client, test_directory) -> pathlib.Path:
|
||||
with downloads_manager_config_context():
|
||||
(test_directory / 'config').mkdir(exist_ok=True)
|
||||
config_path = test_directory / 'config/download_manager.yaml'
|
||||
|
|
@ -187,7 +187,7 @@ async def test_download_manager(
|
|||
test_async_client,
|
||||
test_session, # session is required because downloads can start without the test DB in place.
|
||||
test_download_manager_config,
|
||||
):
|
||||
) -> DownloadManager:
|
||||
# Needed to use signals in test app?
|
||||
api_app.signalize()
|
||||
|
||||
|
|
@ -200,7 +200,7 @@ async def test_download_manager(
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_now():
|
||||
def fake_now() -> Callable:
|
||||
try:
|
||||
set_test_now(datetime(2000, 1, 1))
|
||||
yield set_test_now
|
||||
|
|
@ -210,17 +210,17 @@ def fake_now():
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def successful_download():
|
||||
def successful_download() -> DownloadResult:
|
||||
return DownloadResult(success=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def failed_download():
|
||||
return DownloadResult(error='pytest.fixture failed_download error', success=False)
|
||||
def failed_download() -> DownloadResult:
|
||||
return DownloadResult(error='wrolpi.conftest failed_download error', success=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_download_urls(test_session):
|
||||
def assert_download_urls(test_session) -> Callable[[Set[str]], None]:
|
||||
def asserter(expected_urls: Set[str]):
|
||||
downloads = test_session.query(Download).all()
|
||||
urls = {i.url for i in downloads}
|
||||
|
|
@ -231,7 +231,7 @@ def assert_download_urls(test_session):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_downloads(test_session):
|
||||
def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
def asserter(expected: List[Dict]):
|
||||
|
|
@ -245,7 +245,7 @@ def assert_downloads(test_session):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def test_downloader(test_download_manager):
|
||||
def test_downloader(test_download_manager) -> Downloader:
|
||||
class TestDownloader(Downloader, ABC):
|
||||
"""A testing Downloader"""
|
||||
name = 'test_downloader'
|
||||
|
|
@ -314,7 +314,7 @@ def corrupted_video_file(test_directory) -> pathlib.Path:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def video_file_factory(test_directory):
|
||||
def video_file_factory(test_directory) -> Callable[[pathlib.Path], pathlib.Path]:
|
||||
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
|
||||
|
||||
def _(path: pathlib.Path = None) -> pathlib.Path:
|
||||
|
|
@ -327,7 +327,7 @@ def video_file_factory(test_directory):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def video_bytes():
|
||||
def video_bytes() -> bytes:
|
||||
return (PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()
|
||||
|
||||
|
||||
|
|
@ -467,7 +467,7 @@ def wrol_mode_fixture(test_config, test_download_manager) -> Callable[[bool], Co
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_create_subprocess_shell():
|
||||
def mock_create_subprocess_shell() -> Callable:
|
||||
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
|
||||
async def communicate(*a, **kw):
|
||||
if communicate_side_effect:
|
||||
|
|
@ -507,14 +507,14 @@ def flags_lock():
|
|||
|
||||
|
||||
@pytest.fixture()
|
||||
def tag_factory(test_session):
|
||||
def tag_factory(test_session) -> Callable[[Optional[str]], Awaitable[Tag]]:
|
||||
names = ['one', 'two', 'three', 'four', 'five', 'six']
|
||||
count = 1
|
||||
|
||||
def factory(name: str = None) -> Tag:
|
||||
async def factory(name: str = None) -> Tag:
|
||||
if not name:
|
||||
name = names.pop(0)
|
||||
tag = upsert_tag(name, f'#{str(count) * 6}')
|
||||
tag = await upsert_tag(name, f'#{str(count) * 6}', session=test_session)
|
||||
return tag
|
||||
|
||||
return factory
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ def attach_shared_contexts(app: Sanic):
|
|||
|
||||
# Locks
|
||||
app.shared_ctx.config_save_lock = multiprocessing.Lock()
|
||||
app.shared_ctx.events_lock = multiprocessing.Lock()
|
||||
|
||||
reset_shared_contexts(app)
|
||||
|
||||
|
|
|
|||
|
|
@ -253,6 +253,15 @@ class Download(ModelHelper, Base): # noqa
|
|||
return download
|
||||
raise UnknownDownload(f'Cannot find Download with URL: {url}')
|
||||
|
||||
@classmethod
|
||||
@optional_session
|
||||
def get_all_by_destination(cls, destination: str | pathlib.Path, session: Session = None) -> List['Download']:
|
||||
destination = str(destination)
|
||||
downloads = session.query(Download) \
|
||||
.filter(Download.settings.contains({'destination': destination})) \
|
||||
.all()
|
||||
return downloads
|
||||
|
||||
|
||||
class Downloader:
|
||||
name: str = None
|
||||
|
|
@ -1330,7 +1339,7 @@ class RSSDownloader(Downloader, ABC):
|
|||
# Only download new URLs.
|
||||
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]]
|
||||
|
||||
logger.info(f'Successfully got {len(urls)} urls from rss {download.url}')
|
||||
logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}')
|
||||
|
||||
return DownloadResult(success=True, downloads=urls)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from wrolpi.dates import now
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
HISTORY_SIZE = 100
|
||||
EVENTS_LOCK = multiprocessing.Lock()
|
||||
|
||||
|
||||
class Events:
|
||||
|
|
@ -79,6 +78,14 @@ class Events:
|
|||
def send_shutdown_failed(message: str = None):
|
||||
send_event('shutdown_failed', message, subject='shutdown')
|
||||
|
||||
@staticmethod
|
||||
def send_file_move_completed(message: str = None):
|
||||
send_event('file_move_completed', message, subject='refresh')
|
||||
|
||||
@staticmethod
|
||||
def send_file_move_failed(message: str = None):
|
||||
send_event('file_move_failed', message, subject='refresh')
|
||||
|
||||
|
||||
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
|
||||
log = f'{event=}'
|
||||
|
|
@ -93,7 +100,7 @@ def log_event(event: str, message: str = None, action: str = None, subject: str
|
|||
|
||||
def send_event(event: str, message: str = None, action: str = None, subject: str = None, url: str = None):
|
||||
from wrolpi.api_utils import api_app
|
||||
EVENTS_LOCK.acquire()
|
||||
api_app.shared_ctx.events_lock.acquire()
|
||||
try:
|
||||
# All events will be in time order, they should never be at the exact same time.
|
||||
dt = now()
|
||||
|
|
@ -112,7 +119,7 @@ def send_event(event: str, message: str = None, action: str = None, subject: str
|
|||
while len(api_app.shared_ctx.events_history) > HISTORY_SIZE:
|
||||
api_app.shared_ctx.events_history.pop(0)
|
||||
finally:
|
||||
EVENTS_LOCK.release()
|
||||
api_app.shared_ctx.events_lock.release()
|
||||
|
||||
log_event(event, message, action, subject)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_
|
|||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
|
||||
from . import lib, schema
|
||||
from ..api_utils import json_response, api_app
|
||||
from ..events import Events
|
||||
from ..schema import JSONErrorResponse
|
||||
from ..tags import Tag
|
||||
from ..vars import PYTEST
|
||||
|
|
@ -224,7 +225,9 @@ async def post_move(_: Request, body: schema.Move):
|
|||
|
||||
try:
|
||||
await lib.move(destination, *sources)
|
||||
Events.send_file_move_completed(f'Moving files to {body.destination} succeeded')
|
||||
except Exception as e:
|
||||
Events.send_file_move_failed(f'Moving files to {body.destination} failed!')
|
||||
raise FileConflict(f'Failed to move {sources} to {destination}') from e
|
||||
|
||||
return response.empty(HTTPStatus.NO_CONTENT)
|
||||
|
|
@ -245,7 +248,7 @@ async def post_rename(_: Request, body: schema.Rename):
|
|||
raise FileConflict(f'File already exists: {new_path}')
|
||||
|
||||
try:
|
||||
await lib.rename(path, body.new_name)
|
||||
await lib.rename(path, body.new_name, send_events=True)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to rename {path} to {new_path}', exc_info=e)
|
||||
raise FileConflict(f'Failed to rename {path} to {new_path}') from e
|
||||
|
|
@ -297,14 +300,15 @@ async def post_upload(request: Request):
|
|||
Will not overwrite an existing file, unless a previous upload did not complete."""
|
||||
|
||||
try:
|
||||
destination = request.form['destination'][0]
|
||||
destination_str = request.form['destination'][0]
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to get upload destination', exc_info=e)
|
||||
raise UnknownDirectory('Must provide destination') from e
|
||||
|
||||
destination = get_media_directory() / destination
|
||||
if not destination.is_dir():
|
||||
raise UnknownDirectory(f'Destination must be a relative directory that is already in the media directory: {destination}')
|
||||
destination = get_media_directory() / destination_str
|
||||
if destination_str.startswith('/') or destination_str.startswith('.') or not destination.is_dir():
|
||||
msg = f'Destination must be a relative directory that is already in the media directory: {destination}'
|
||||
raise UnknownDirectory(msg)
|
||||
|
||||
try:
|
||||
filename = str(request.form['filename'][0])
|
||||
|
|
@ -385,7 +389,11 @@ async def post_upload(request: Request):
|
|||
logger.info(f'Got final chunk of uploaded file: {output_str}')
|
||||
del api_app.shared_ctx.uploaded_files[output_str]
|
||||
# Upsert this new file (and any related files) into the DB.
|
||||
background_task(lib.upsert_file(output, tag_names=tag_names))
|
||||
coro = lib.upsert_file(output, tag_names=tag_names)
|
||||
if PYTEST:
|
||||
await coro
|
||||
else:
|
||||
background_task(coro)
|
||||
return response.empty(HTTPStatus.CREATED)
|
||||
|
||||
# Request the next chunk.
|
||||
|
|
|
|||
|
|
@ -9,13 +9,15 @@ import pathlib
|
|||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from collections import OrderedDict
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Union, Dict, Generator, Iterable
|
||||
from typing import List, Tuple, Union, Dict, Generator, Iterable, Set
|
||||
|
||||
import cachetools.func
|
||||
import psycopg2
|
||||
from sqlalchemy import asc
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi import flags
|
||||
|
|
@ -24,7 +26,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
|
|||
partition, cancelable_wrapper, \
|
||||
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
|
||||
get_wrolpi_config, \
|
||||
timer
|
||||
timer, chunks, unique_by_predicate
|
||||
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
|
||||
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
|
||||
|
|
@ -32,7 +34,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
|
|||
from wrolpi.events import Events
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
from wrolpi.lang import ISO_639_CODES
|
||||
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where
|
||||
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
try:
|
||||
|
|
@ -48,7 +50,7 @@ logger = logger.getChild(__name__)
|
|||
|
||||
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
|
||||
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
|
||||
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file',
|
||||
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
|
||||
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
|
||||
|
||||
|
||||
|
|
@ -311,6 +313,12 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = Fals
|
|||
return path.name, ''
|
||||
|
||||
|
||||
def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \
|
||||
-> List[pathlib.Path]:
|
||||
"""Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem."""
|
||||
return unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0])
|
||||
|
||||
|
||||
refresh_logger = logger.getChild('refresh')
|
||||
|
||||
|
||||
|
|
@ -578,6 +586,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
|
|||
refreshing_all_files = False
|
||||
|
||||
with flags.refreshing, timer('refresh_files', 'info'):
|
||||
api_app.shared_ctx.refresh['counted_files'] = 0
|
||||
if not paths:
|
||||
refresh_logger.warning('Refreshing all files')
|
||||
refreshing_all_files = True
|
||||
|
|
@ -603,8 +612,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
|
|||
continue
|
||||
directories.extend(dirs)
|
||||
found_directories |= set(dirs)
|
||||
api_app.shared_ctx.refresh['counted_files'] = api_app.shared_ctx.refresh.get('counted_files',
|
||||
0) + len(files)
|
||||
add_files_to_refresh_count(len(files))
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
refresh_logger.info(f'Counted {api_app.shared_ctx.refresh["counted_files"]} files')
|
||||
|
|
@ -633,7 +641,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
|
|||
with get_db_session() as session:
|
||||
parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))}
|
||||
parent_directories |= set(filter(lambda i: i.is_dir(), paths))
|
||||
await upsert_directories(parent_directories, found_directories)
|
||||
upsert_directories(parent_directories, found_directories)
|
||||
|
||||
if send_events:
|
||||
Events.send_global_after_refresh_completed()
|
||||
|
|
@ -684,7 +692,7 @@ async def apply_indexers():
|
|||
break
|
||||
|
||||
|
||||
async def upsert_directories(parent_directories, directories):
|
||||
def upsert_directories(parent_directories, directories):
|
||||
"""
|
||||
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
|
||||
`directories`.
|
||||
|
|
@ -1168,7 +1176,17 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
|
|||
return 0
|
||||
|
||||
|
||||
def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
|
||||
MOVE_CHUNK_SIZE = 100
|
||||
|
||||
|
||||
def add_files_to_refresh_count(count: int):
|
||||
from wrolpi.api_utils import api_app
|
||||
count = api_app.shared_ctx.refresh.get('counted_files', 0) + count
|
||||
api_app.shared_ctx.refresh['counted_files'] = count
|
||||
|
||||
|
||||
async def _move(destination: pathlib.Path, *sources: pathlib.Path, session: Session) \
|
||||
-> OrderedDict[pathlib.Path, pathlib.Path]:
|
||||
media_directory = get_media_directory()
|
||||
for source in sources:
|
||||
if not str(source).startswith(str(media_directory)):
|
||||
|
|
@ -1177,70 +1195,155 @@ def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathl
|
|||
raise FileNotFoundError(f'{destination} is not within the media directory')
|
||||
|
||||
destination_existed = destination.is_dir()
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# [ (pathlib.Path('source'), pathlib.Path('destination')), ... ]
|
||||
plan = list()
|
||||
unique_paths = set()
|
||||
# Get all files that need to be moved. Check for destination conflicts.
|
||||
for source in sources:
|
||||
if source.is_dir():
|
||||
# Recursively move the directory's files.
|
||||
for path in walk(source):
|
||||
# Preserve the directory structure that contains this file.
|
||||
new_file = destination / source.name / path.relative_to(source)
|
||||
if new_file.exists():
|
||||
raise FileConflict(f'Cannot move {path} to {new_file} because it already exists')
|
||||
if path not in unique_paths:
|
||||
plan.append((path, new_file))
|
||||
unique_paths.add(path)
|
||||
elif source.is_file():
|
||||
# Move all files associated with the source file.
|
||||
for file in glob_shared_stem(source):
|
||||
new_file = destination / file.name
|
||||
if new_file.exists():
|
||||
raise FileConflict(f'Cannot move {file} to {new_file} because it already exists')
|
||||
if file not in unique_paths:
|
||||
plan.append((file, new_file))
|
||||
unique_paths.add(file)
|
||||
# The files that will be moved. [ (old_file, new_file), ... ]
|
||||
plan: Dict[pathlib.Path, pathlib.Path] = dict()
|
||||
|
||||
logger.info(f'Executing move plan with {len(plan)} steps.')
|
||||
def add_file_group_to_plan(file_group_: FileGroup):
|
||||
primary_path_ = file_group_.primary_path
|
||||
if primary_path_ not in plan:
|
||||
new_primary_path_ = destination / primary_path_.name
|
||||
if new_primary_path_.exists():
|
||||
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
|
||||
plan[primary_path_] = new_primary_path_
|
||||
|
||||
# Track the steps performed so they can be reversed.
|
||||
revert_plan = list()
|
||||
def add_source_file_group_to_plan(file_group_: FileGroup, source_: pathlib.Path):
|
||||
primary_path_ = file_group_.primary_path
|
||||
if primary_path_ not in plan:
|
||||
new_primary_path_ = destination / source_.name / primary_path_.relative_to(source)
|
||||
if new_primary_path_.exists():
|
||||
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
|
||||
plan[primary_path_] = new_primary_path_
|
||||
|
||||
def do_plan(plan_):
|
||||
# Move deepest files first.
|
||||
plan_ = sorted(plan_, key=lambda i: (len(i[0].parents), i[0].name), reverse=True)
|
||||
for old_file, new_file in plan_: # noqa
|
||||
new_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug(f'Moving {old_file} to {new_file}')
|
||||
# if not new_file.exists():
|
||||
shutil.move(str(old_file), str(new_file))
|
||||
logger.debug(f'Moved {old_file} to {new_file}')
|
||||
revert_plan.append((new_file, old_file))
|
||||
# Move is complete.
|
||||
return plan_
|
||||
# Sources may be a list of files, we only want to issue a query for one file in each FileGroup.
|
||||
sources = get_unique_files_by_stem(sources)
|
||||
logger.info(f'move got {len(sources)} to move')
|
||||
|
||||
try:
|
||||
plan = do_plan(plan)
|
||||
with flags.refresh_counting:
|
||||
for source in sources:
|
||||
if source.is_dir():
|
||||
delete_directory(source)
|
||||
logger.info(f'Move execution completed')
|
||||
return plan
|
||||
except Exception as e:
|
||||
logger.error(f'Move failed', exc_info=e)
|
||||
# Move files back.
|
||||
do_plan(revert_plan)
|
||||
# Delete the directories that were created.
|
||||
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
|
||||
for path in directories:
|
||||
if path.is_dir():
|
||||
delete_directory(path)
|
||||
# Remove destination only if it did not exist at the start.
|
||||
if destination.is_dir() and destination_existed is False:
|
||||
delete_directory(destination)
|
||||
raise
|
||||
if source.is_file():
|
||||
# Get any FileGroups that share the source's stem.
|
||||
files = glob_shared_stem(source)
|
||||
file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(files)).all()
|
||||
if not file_groups:
|
||||
# No FileGroups for this source file, create one.
|
||||
try:
|
||||
fg = FileGroup.from_paths(session, *files)
|
||||
file_groups = [fg, ]
|
||||
except NoPrimaryFile:
|
||||
for file in files:
|
||||
fg = FileGroup.from_paths(session, file)
|
||||
add_file_group_to_plan(fg)
|
||||
add_files_to_refresh_count(len(file_groups))
|
||||
for fg in file_groups:
|
||||
add_file_group_to_plan(fg)
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
elif source.is_dir():
|
||||
stmt = 'SELECT primary_path FROM file_group WHERE primary_path LIKE :like'
|
||||
for (primary_path,) in session.execute(stmt, dict(like=f'{source}%')):
|
||||
primary_path = pathlib.Path(primary_path)
|
||||
if primary_path not in plan:
|
||||
new_primary_path = destination / source.name / primary_path.relative_to(source)
|
||||
if new_primary_path.exists():
|
||||
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path}')
|
||||
plan[primary_path] = new_primary_path
|
||||
add_files_to_refresh_count(1)
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# Move directories of this source.
|
||||
for directory in (i for i in walk(source) if i.is_dir()):
|
||||
new_directory = destination / source.name / directory.relative_to(source)
|
||||
plan[directory] = new_directory
|
||||
|
||||
# Find any files in this directory that are not yet in the DB and add them to the plan.
|
||||
files = {i for i in walk(source) if i.is_file()}
|
||||
if missing_files := files - set(plan.keys()):
|
||||
for paths in group_files_by_stem(missing_files):
|
||||
try:
|
||||
# Only create the FileGroup if a primary_path can be found.
|
||||
get_primary_file(paths)
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
add_files_to_refresh_count(1)
|
||||
add_source_file_group_to_plan(file_group, source)
|
||||
except NoPrimaryFile:
|
||||
# No primary path for these paths, create FileGroups for each.
|
||||
add_files_to_refresh_count(1)
|
||||
for file in paths:
|
||||
fg = FileGroup.from_paths(session, file)
|
||||
add_source_file_group_to_plan(fg, source)
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
else:
|
||||
raise UnknownFile(f'Unknown path type, cannot move: {source}')
|
||||
|
||||
# Sort plan by the deepest files first.
|
||||
plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True))
|
||||
|
||||
# Revert plan is built out as files are moved.
|
||||
revert_plan = OrderedDict()
|
||||
new_directories = set()
|
||||
|
||||
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path]):
|
||||
"""Apply the move plan to all the FileGroups.
|
||||
|
||||
@warning: Cannot be cancelled!"""
|
||||
# Move FileGroups in groups.
|
||||
for chunk in chunks(plan_.items(), MOVE_CHUNK_SIZE):
|
||||
old_paths = [i for i, j in chunk]
|
||||
old_files, old_directories = partition(lambda i: i.is_file(), old_paths)
|
||||
file_groups_ = session.query(FileGroup).filter(FileGroup.primary_path.in_(old_files)).all()
|
||||
if len(file_groups_) != len(old_files):
|
||||
# We ensured there are FileGroups while building the plan, maybe user deleted a file?
|
||||
raise RuntimeError('Could not get all FileGroups for move')
|
||||
|
||||
for file_group_ in file_groups_:
|
||||
old_file = file_group_.primary_path
|
||||
new_primary_path_ = plan_[old_file]
|
||||
parent = new_primary_path_.parent
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
if parent not in new_directories:
|
||||
new_directories.add(parent)
|
||||
file_group_.move(new_primary_path_)
|
||||
revert_plan[new_primary_path_] = old_file
|
||||
for directory_ in old_directories:
|
||||
delete_directory(directory_)
|
||||
|
||||
existing_directories = {i[0] for i in session.query(Directory.path)}
|
||||
missing_directories = new_directories - existing_directories
|
||||
for directory_ in missing_directories:
|
||||
session.add(Directory(path=directory_, name=directory_.name))
|
||||
|
||||
# Don't forget what has moved.
|
||||
session.flush(file_groups_)
|
||||
|
||||
with flags.refresh_discovery:
|
||||
try:
|
||||
do_plan(plan)
|
||||
for source in sources:
|
||||
if source.is_dir():
|
||||
delete_directory(source)
|
||||
logger.info(f'Move execution completed')
|
||||
except Exception as e:
|
||||
logger.error(f'Move failed', exc_info=e)
|
||||
new_directories = set()
|
||||
# Move files back. Get copy because do_plan will change the revert plan.
|
||||
do_plan(revert_plan.copy())
|
||||
# Delete the directories that were created.
|
||||
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
|
||||
for path in directories:
|
||||
if path.is_dir():
|
||||
delete_directory(path)
|
||||
# Remove destination only if it did not exist at the start.
|
||||
if destination.is_dir() and destination_existed is False:
|
||||
delete_directory(destination)
|
||||
raise
|
||||
|
||||
return plan
|
||||
|
||||
|
||||
def delete_directory(directory: pathlib.Path, recursive: bool = False):
|
||||
|
|
@ -1263,41 +1366,37 @@ def delete_directory(directory: pathlib.Path, recursive: bool = False):
|
|||
curs.execute(stmt, (str(directory),))
|
||||
|
||||
|
||||
async def move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
|
||||
@optional_session
|
||||
async def move(destination: pathlib.Path, *sources: pathlib.Path, session: Session = None) \
|
||||
-> OrderedDict[pathlib.Path, pathlib.Path]:
|
||||
"""Moves a file or directory (recursively), preserving applied Tags.
|
||||
|
||||
If the move fails at any point, the files will be returned to their previous locations.
|
||||
|
||||
Returns a list of tuples, the first Path in the tuple is the file's old location, the second Path is where the file
|
||||
was moved.
|
||||
Returns an OrderedDict, the key is the file's old location, the value is where the file was moved.
|
||||
"""
|
||||
# Move the files first, if this fails it will revert itself and raise an error.
|
||||
plan = _move(destination, *sources)
|
||||
if not session:
|
||||
raise RuntimeError('`session` is required')
|
||||
|
||||
# Files were moved, migrate any Tags.
|
||||
plan_map = {old: new for old, new in plan}
|
||||
old_file_paths = list(plan_map.keys())
|
||||
with get_db_session(commit=True) as session:
|
||||
old_files: List[Tuple[FileGroup, TagFile]] = session.query(FileGroup, TagFile) \
|
||||
.filter(FileGroup.primary_path.in_(old_file_paths)) \
|
||||
.outerjoin(TagFile).all()
|
||||
logger.info(f'Moving {len(old_files)} tagged files')
|
||||
for old_file, _ in old_files:
|
||||
try:
|
||||
new_file_path = plan_map[old_file.primary_path]
|
||||
logger.info(f'Moving tagged file {old_file.primary_path} to {new_file_path}')
|
||||
old_file.move(new_file_path, move_files=False)
|
||||
except KeyError as e:
|
||||
# Got a duplicate?
|
||||
logger.error('Error during tag migration', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
session.commit()
|
||||
with flags.refreshing:
|
||||
from wrolpi.api_utils import api_app
|
||||
api_app.shared_ctx.refresh['counted_files'] = 0
|
||||
|
||||
if PYTEST:
|
||||
await refresh_files([destination, *sources])
|
||||
else:
|
||||
background_task(refresh_files([destination, *sources]))
|
||||
# Move the files, if this fails it will revert itself and raise an error.
|
||||
with timer('moving files', 'info'):
|
||||
plan = await _move(destination, *sources, session=session)
|
||||
session.commit()
|
||||
|
||||
with flags.refresh_indexing:
|
||||
await apply_indexers()
|
||||
|
||||
with flags.refresh_modeling:
|
||||
await apply_modelers()
|
||||
|
||||
with flags.refresh_cleanup:
|
||||
await apply_refresh_cleanup()
|
||||
# Save tags now that files have been moved.
|
||||
save_tags_config()
|
||||
|
||||
return plan
|
||||
|
||||
|
|
@ -1320,25 +1419,40 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
|
|||
return new_path
|
||||
|
||||
|
||||
async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path:
|
||||
async def rename_directory(directory: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) \
|
||||
-> pathlib.Path:
|
||||
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
|
||||
if not session:
|
||||
raise RuntimeError('`session` is required')
|
||||
|
||||
new_directory = directory.with_name(new_name)
|
||||
if new_directory.exists():
|
||||
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
|
||||
|
||||
# Move all paths into the new directory.
|
||||
paths = list(directory.iterdir())
|
||||
await move(new_directory, *paths)
|
||||
try:
|
||||
await move(new_directory, *paths, session=session)
|
||||
if send_events:
|
||||
Events.send_file_move_completed(f'Directory has been renamed: {directory}')
|
||||
except Exception:
|
||||
if send_events:
|
||||
Events.send_file_move_failed(f'Directory rename has failed: {directory}')
|
||||
raise
|
||||
# Remove the old directory.
|
||||
delete_directory(directory)
|
||||
|
||||
return new_directory
|
||||
|
||||
|
||||
async def rename(path: pathlib.Path, new_name: str) -> pathlib.Path:
|
||||
@optional_session
|
||||
async def rename(path: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) -> pathlib.Path:
|
||||
"""Rename a directory or file. Preserve any tags."""
|
||||
if not session:
|
||||
raise RuntimeError('`session` is required')
|
||||
|
||||
if path.is_dir():
|
||||
return await rename_directory(path, new_name)
|
||||
return await rename_directory(path, new_name, session=session, send_events=send_events)
|
||||
|
||||
return await rename_file(path, new_name)
|
||||
|
||||
|
|
@ -1399,26 +1513,36 @@ async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_nam
|
|||
paths = glob_shared_stem(pathlib.Path(file))
|
||||
# Remove any ignored files.
|
||||
paths = remove_files_in_ignored_directories(paths)
|
||||
if paths:
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
logger.debug(f'upsert_file: {file_group}')
|
||||
# Re-index the contents of the file.
|
||||
session.add(file_group)
|
||||
session.flush([file_group, ])
|
||||
|
||||
try:
|
||||
file_group.do_model(session)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
for tag_name in (tag_names or []):
|
||||
if tag_name not in file_group.tag_names:
|
||||
tag = Tag.get_by_name(tag_name)
|
||||
file_group.add_tag(tag.id)
|
||||
|
||||
session.commit()
|
||||
return file_group
|
||||
else:
|
||||
if not paths:
|
||||
logger.warning('upsert_file called, but all files are in ignored directories!')
|
||||
return
|
||||
|
||||
for i in range(2):
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
# Re-index the contents of the file.
|
||||
try:
|
||||
session.flush([file_group, ])
|
||||
break
|
||||
except IntegrityError:
|
||||
# Another process inserted this FileGroup.
|
||||
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
|
||||
session.rollback()
|
||||
continue
|
||||
else:
|
||||
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
|
||||
|
||||
logger.debug(f'upsert_file: {file_group}')
|
||||
try:
|
||||
file_group.do_model(session)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
for tag_name in (tag_names or []):
|
||||
if tag_name not in file_group.tag_names:
|
||||
tag = Tag.get_by_name(tag_name)
|
||||
file_group.add_tag(tag.id)
|
||||
|
||||
session.commit()
|
||||
return file_group
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from sqlalchemy import types
|
|||
from sqlalchemy.orm import deferred, relationship, Session
|
||||
|
||||
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
|
||||
get_relative_to_media_directory
|
||||
get_relative_to_media_directory, unique_by_predicate
|
||||
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.errors import FileGroupIsTagged, UnknownFile
|
||||
|
|
@ -154,9 +154,8 @@ class FileGroup(ModelHelper, Base):
|
|||
"""Add all `paths` to this FileGroup.files."""
|
||||
from wrolpi.files.lib import get_mimetype
|
||||
new_files = list(self.files) if self.files else list()
|
||||
for file in paths:
|
||||
new_files.append(dict(path=file, mimetype=get_mimetype(file)))
|
||||
self.files = new_files
|
||||
new_files.extend([dict(path=path, mimetype=get_mimetype(path)) for path in paths])
|
||||
self.files = unique_by_predicate(new_files, lambda i: i['path'])
|
||||
|
||||
def my_files(self, *mimetypes: str) -> List[dict]:
|
||||
"""Return all files related to this group that match any of the provided mimetypes.
|
||||
|
|
@ -225,7 +224,9 @@ class FileGroup(ModelHelper, Base):
|
|||
|
||||
if session := Session.object_session(self):
|
||||
from wrolpi.downloader import download_manager
|
||||
if self.url and add_to_skip_list is True:
|
||||
if self.url and (download := download_manager.get_download(session, self.url)):
|
||||
download.delete(add_to_skip_list)
|
||||
elif self.url and add_to_skip_list is True:
|
||||
download_manager.add_to_skip_list(self.url)
|
||||
session.delete(self)
|
||||
|
||||
|
|
@ -254,25 +255,26 @@ class FileGroup(ModelHelper, Base):
|
|||
@classmethod
|
||||
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
|
||||
"""Create a new FileGroup which contains the provided file paths."""
|
||||
from wrolpi.files.lib import get_primary_file, get_mimetype, split_path_stem_and_suffix
|
||||
from wrolpi.files.lib import get_primary_file, get_mimetype, get_unique_files_by_stem
|
||||
|
||||
unique_stems = {split_path_stem_and_suffix(i, full=True)[0] for i in paths}
|
||||
unique_stems = get_unique_files_by_stem(paths)
|
||||
if len(unique_stems) > 1:
|
||||
raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem.')
|
||||
|
||||
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
|
||||
logger.debug(f'FileGroup.from_paths: {len(existing_groups)}')
|
||||
if len(existing_groups) == 0:
|
||||
# These paths have not been used previously, create a new FileGroup.
|
||||
file_group = FileGroup()
|
||||
primary_path = get_primary_file(paths)
|
||||
file_group.primary_path = primary_path
|
||||
file_group.append_files(*paths)
|
||||
file_group.mimetype = get_mimetype(file_group.primary_path)
|
||||
session.add(file_group)
|
||||
elif len(existing_groups) == 1:
|
||||
# Found one FileGroup with these paths, no need to create a new FileGroup.
|
||||
file_group = existing_groups[0]
|
||||
primary_path = file_group.primary_path
|
||||
file_group.append_files(*paths)
|
||||
primary_path = get_primary_file(file_group.my_paths())
|
||||
else:
|
||||
# Multiple FileGroups contain these paths as primary.
|
||||
primary_path = get_primary_file(paths)
|
||||
|
|
@ -284,8 +286,11 @@ class FileGroup(ModelHelper, Base):
|
|||
if not isinstance(primary_path, pathlib.Path):
|
||||
raise ValueError('Cannot create FileGroup without a primary path.')
|
||||
|
||||
file_group.primary_path = primary_path
|
||||
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
|
||||
file_group.size = sum(i.stat().st_size for i in paths)
|
||||
file_group.mimetype = get_mimetype(file_group.primary_path)
|
||||
logger.debug(f'FileGroup.from_paths: {file_group}')
|
||||
|
||||
return file_group
|
||||
|
||||
|
|
@ -329,28 +334,44 @@ class FileGroup(ModelHelper, Base):
|
|||
|
||||
self.files = collected_files
|
||||
|
||||
def move(self, new_primary_path: pathlib.Path, move_files: bool = True):
|
||||
def move(self, new_primary_path: pathlib.Path):
|
||||
"""Move all files in this group to a new location."""
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
if move_files and new_primary_path.exists():
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, glob_shared_stem
|
||||
|
||||
if new_primary_path.exists():
|
||||
raise FileExistsError(f'Cannot move {self} to {new_primary_path} because it already exists.')
|
||||
|
||||
# Adopt any new files when a FileGroup has multiple files.
|
||||
if len(self.my_files()) > 1:
|
||||
self.append_files(*glob_shared_stem(self.primary_path))
|
||||
|
||||
new_name, _ = split_path_stem_and_suffix(new_primary_path, full=True)
|
||||
# Need a deepcopy because changes to self.files are ignored otherwise.
|
||||
new_files = copy.deepcopy(self.files)
|
||||
# Ensure that all destination files do not yet exist before move.
|
||||
for file in new_files:
|
||||
_, suffix = split_path_stem_and_suffix(file['path'])
|
||||
new_path = pathlib.Path(f'{new_name}{suffix}')
|
||||
if new_path.exists():
|
||||
raise FileExistsError(f'Cannot move {self} to {new_path} because it already exists.')
|
||||
|
||||
# Collect all files that exist and move them.
|
||||
existing_files = list()
|
||||
for idx, file in enumerate(new_files):
|
||||
_, suffix = split_path_stem_and_suffix(file['path'])
|
||||
new_path = pathlib.Path(f'{new_name}{suffix}')
|
||||
if move_files:
|
||||
if pathlib.Path(file['path']).is_file():
|
||||
shutil.move(file['path'], new_path)
|
||||
new_files[idx]['path'] = new_path
|
||||
self.files = new_files
|
||||
existing_files.append({'path': new_path, 'mimetype': file['mimetype']})
|
||||
|
||||
logger.debug(f'Moved FileGroup: {self.primary_path} -> {new_primary_path}')
|
||||
self.files = existing_files
|
||||
if self.title == self.primary_path.name:
|
||||
# Do not overwrite title from modeler.
|
||||
self.title = new_primary_path.name
|
||||
self.primary_path = new_primary_path
|
||||
# Need to re-index for self.data.
|
||||
self.indexed = False
|
||||
self.indexed = False if self.data else self.indexed
|
||||
# Flush the changes to the FileGroup.
|
||||
self.flush()
|
||||
|
||||
|
|
@ -384,11 +405,15 @@ class FileGroup(ModelHelper, Base):
|
|||
if model_class := self.get_model_class():
|
||||
if model := model_class.get_by_path(self.primary_path, session):
|
||||
# Already modeled.
|
||||
logger.debug(f'already modeled: {self} {model}')
|
||||
return model
|
||||
# Create new model (Video/Archive/Ebook).
|
||||
return model_class.do_model(self, session)
|
||||
model = model_class.do_model(self, session)
|
||||
logger.debug(f'new model: {self} {model}')
|
||||
return model
|
||||
|
||||
# Index the file if no models are available.
|
||||
logger.debug(f'no model class: {self}')
|
||||
self.do_index()
|
||||
self.indexed = True
|
||||
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_search_any_tag(test_async_client, test_session, make_files_structure, tag_factory):
|
||||
one, two = tag_factory(), tag_factory()
|
||||
one, two = await tag_factory(), await tag_factory()
|
||||
files = [
|
||||
'foo.txt',
|
||||
'foo bar.txt',
|
||||
|
|
@ -299,34 +299,35 @@ def test_file_statistics(test_session, test_client, test_directory, example_pdf,
|
|||
}
|
||||
|
||||
|
||||
def test_file_group_tag_by_primary_path(test_session, test_client, test_directory, example_singlefile, tag_factory,
|
||||
insert_file_group):
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_tag_by_primary_path(test_session, test_async_client, test_directory, example_singlefile,
|
||||
tag_factory, insert_file_group):
|
||||
singlefile = FileGroup.from_paths(test_session, example_singlefile)
|
||||
tag1 = tag_factory()
|
||||
tag2 = tag_factory()
|
||||
tag1 = await tag_factory()
|
||||
tag2 = await tag_factory()
|
||||
test_session.commit()
|
||||
|
||||
# FileGroup can be tagged with its primary_path.
|
||||
content = dict(file_group_primary_path=str(singlefile.primary_path.relative_to(test_directory)), tag_name=tag1.name)
|
||||
request, response = test_client.post('/api/files/tag', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
assert test_session.query(TagFile).count() == 1
|
||||
|
||||
# FileGroup can be tagged with its id.
|
||||
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
|
||||
request, response = test_client.post('/api/files/tag', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
assert test_session.query(TagFile).count() == 2
|
||||
|
||||
# FileGroup can be untagged with its primary_path.
|
||||
content = dict(file_group_id=singlefile.id, tag_id=tag1.id)
|
||||
request, response = test_client.post('/api/files/untag', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert test_session.query(TagFile).count() == 1
|
||||
|
||||
# FileGroup can be untagged with its id.
|
||||
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
|
||||
request, response = test_client.post('/api/files/untag', content=json.dumps(content))
|
||||
request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert test_session.query(TagFile).count() == 0
|
||||
|
||||
|
|
@ -468,11 +469,13 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f
|
|||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
|
||||
def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form, tag_factory,
|
||||
video_bytes):
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_upload(test_session, test_async_client, test_directory, make_files_structure, make_multipart_form,
|
||||
tag_factory,
|
||||
video_bytes):
|
||||
"""A file can be uploaded in chunks directly to the destination."""
|
||||
make_files_structure(['uploads/'])
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
|
||||
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
|
||||
forms = [
|
||||
|
|
@ -487,7 +490,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
|
|||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = test_client.post('/api/files/upload', content=body, headers=headers)
|
||||
request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.OK, response.content.decode()
|
||||
|
||||
output = test_directory / 'uploads/video.mp4'
|
||||
|
|
@ -505,7 +508,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
|
|||
dict(name='chunk', value=part2, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
request, response = test_client.post('/api/files/upload', content=body, headers=headers)
|
||||
request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
|
||||
|
||||
assert output.is_file()
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ async def test_discover_calibre_cover(test_session, test_directory, example_epub
|
|||
@pytest.mark.asyncio
|
||||
async def test_move_ebook(test_async_client, test_session, test_directory, example_epub, image_file, tag_factory):
|
||||
"""An ebook is re-indexed when moved."""
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
shutil.move(image_file, example_epub.with_suffix('.jpg'))
|
||||
await refresh_files()
|
||||
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ async def test_delete_file_names(test_async_client, test_session, make_files_str
|
|||
})
|
||||
foo_fg = FileGroup.from_paths(test_session, foo)
|
||||
foo1_fg = FileGroup.from_paths(test_session, foo1)
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
foo1_fg.add_tag(tag.id)
|
||||
test_session.commit()
|
||||
assert foo.is_file()
|
||||
|
|
@ -106,7 +106,7 @@ async def test_delete_file_link(test_async_client, test_session, test_directory)
|
|||
@pytest.mark.asyncio
|
||||
async def test_delete_tagged(test_async_client, test_session, make_files_structure, tag_factory, video_bytes):
|
||||
"""Cannot delete a file that has been tagged."""
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
make_files_structure({'foo/bar.txt': 'asdf', 'foo/bar.mp4': video_bytes})
|
||||
await lib.refresh_files()
|
||||
# Both files end up in a group.
|
||||
|
|
@ -365,7 +365,7 @@ async def test_file_group_tag(test_session, make_files_structure, test_directory
|
|||
"""A FileGroup can be tagged."""
|
||||
make_files_structure(['foo.mp4'])
|
||||
await lib.refresh_discover_paths([test_directory, ], now())
|
||||
one = tag_factory()
|
||||
one = await tag_factory()
|
||||
|
||||
foo: FileGroup = test_session.query(FileGroup).one()
|
||||
foo.add_tag(one.id)
|
||||
|
|
@ -401,7 +401,7 @@ async def test_refresh_many_files(test_async_client, test_session, make_files_st
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_cancel(test_session, make_files_structure, test_directory):
|
||||
async def test_refresh_cancel(test_async_client, test_session, make_files_structure, test_directory):
|
||||
"""Refresh tasks can be canceled."""
|
||||
# Creat a lot of files so the refresh will take too long.
|
||||
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
|
||||
|
|
@ -723,15 +723,16 @@ async def test_refresh_directories(test_session, test_directory, assert_director
|
|||
assert_directories({'foo', 'baz'})
|
||||
|
||||
|
||||
def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory,
|
||||
video_bytes, srt_file3):
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory,
|
||||
video_bytes, srt_file3):
|
||||
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
|
||||
migrated."""
|
||||
vid, srt = make_files_structure({
|
||||
'vid.mp4': video_bytes,
|
||||
'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(),
|
||||
})
|
||||
one, two = tag_factory(), tag_factory()
|
||||
one, two = await tag_factory(), await tag_factory()
|
||||
vid_group = FileGroup.from_paths(test_session, vid)
|
||||
srt_group = FileGroup.from_paths(test_session, srt)
|
||||
test_session.add_all([vid_group, srt_group])
|
||||
|
|
@ -774,10 +775,11 @@ async def test_move(test_async_client, test_session, test_directory, make_files_
|
|||
})
|
||||
foo = test_directory / 'foo'
|
||||
qux = test_directory / 'qux'
|
||||
await lib.refresh_files()
|
||||
|
||||
# mv foo qux
|
||||
plan = await lib.move(qux, foo)
|
||||
plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan]
|
||||
plan = await lib.move(qux, foo, session=test_session)
|
||||
plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan.items()]
|
||||
# The deepest files are moved first.
|
||||
assert plan == [('foo/bar/baz/archive.html', 'qux/foo/bar/baz/archive.html'),
|
||||
('foo/bar/video.mp4', 'qux/foo/bar/video.mp4'),
|
||||
|
|
@ -805,10 +807,16 @@ async def test_move_files(test_async_client, test_session, test_directory, make_
|
|||
})
|
||||
dest = test_directory / 'dest'
|
||||
|
||||
await lib.move(dest, one, two)
|
||||
plan = await lib.move(dest, one, two, session=test_session)
|
||||
assert list(plan.items()) == [
|
||||
(test_directory / 'foo/one.txt', test_directory / 'dest/one.txt'),
|
||||
(test_directory / 'two.txt', test_directory / 'dest/two.txt'),
|
||||
]
|
||||
# one.txt is moved out of foo.
|
||||
assert (dest / 'one.txt').read_text() == 'one'
|
||||
assert (dest / 'two.txt').read_text() == 'two'
|
||||
assert not (test_directory / 'foo/one.txt').exists(), 'one.txt is lingering in source'
|
||||
assert not (test_directory / 'two.txt').exists(), 'two.txt is lingering in source'
|
||||
assert (dest / 'one.txt').read_text() == 'one', 'one.txt was not moved to destination'
|
||||
assert (dest / 'two.txt').read_text() == 'two', 'two.txt was not moved to destination'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -830,8 +838,14 @@ async def test_move_deep_directory(test_async_client, test_session, test_directo
|
|||
bar, qux = baz.parent, quux.parent
|
||||
dest = test_directory / 'deep/dest'
|
||||
|
||||
# mv foo/bar foo/qux foo/quuz.mp4 deep/dest
|
||||
await lib.move(dest, bar, qux, quuz_mp4)
|
||||
# mv foo/bar foo/qux foo/quuz.mp4 foo/quuz.txt deep/dest
|
||||
plan = await lib.move(dest, bar, qux, quuz_mp4, quuz_txt, session=test_session)
|
||||
assert list(plan.items()) == [
|
||||
(test_directory / 'foo/qux/quux.txt', test_directory / 'deep/dest/qux/quux.txt'),
|
||||
(test_directory / 'foo/bar/baz.txt', test_directory / 'deep/dest/bar/baz.txt'),
|
||||
(test_directory / 'foo/quuz.txt', test_directory / 'deep/dest/quuz.txt'),
|
||||
(test_directory / 'foo/quuz.mp4', test_directory / 'deep/dest/quuz.mp4'),
|
||||
]
|
||||
|
||||
# Unrelated files and directories are untouched.
|
||||
assert not (test_directory / 'deep/dest/foo').is_dir(), 'foo/ should not have been moved.'
|
||||
|
|
@ -861,7 +875,7 @@ async def test_move_directory(test_async_client, test_session, test_directory, m
|
|||
assert_directories({'foo'})
|
||||
|
||||
bar = test_directory / 'bar'
|
||||
await lib.rename(test_directory / 'foo', 'bar')
|
||||
await lib.rename(test_directory / 'foo', 'bar', session=test_session)
|
||||
assert (bar / 'one.txt').read_text() == 'one'
|
||||
assert_directories({'bar'})
|
||||
|
||||
|
|
@ -890,7 +904,7 @@ async def test_move_error(test_session, test_directory, make_files_structure, vi
|
|||
return shutil.move(*args, **kwargs)
|
||||
|
||||
with mock.patch('wrolpi.files.lib.shutil.move', mock_shutil_move), pytest.raises(FileNotFoundError):
|
||||
await lib.move(qux, foo)
|
||||
await lib.move(qux, foo, session=test_session)
|
||||
# The move failed, the files should be moved back.
|
||||
# foo was not deleted.
|
||||
assert foo.is_dir()
|
||||
|
|
@ -923,21 +937,19 @@ async def test_file_group_move(test_async_client, test_session, make_files_struc
|
|||
assert new_srt.read_text() == srt_text
|
||||
assert not video.is_file()
|
||||
assert not srt.is_file()
|
||||
# Moved files must be re-indexed.
|
||||
assert file_group.indexed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_tagged(test_async_client, test_session, test_directory, make_files_structure, tag_factory):
|
||||
"""A FileGroup's tag is preserved when moved or renamed."""
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
foo, bar, = make_files_structure({
|
||||
'foo/foo.txt': 'foo',
|
||||
'foo/bar.txt': 'bar',
|
||||
})
|
||||
await lib.refresh_files()
|
||||
bar_file_group, foo_file_group = test_session.query(FileGroup).order_by(FileGroup.primary_path)
|
||||
foo_file_group.title = 'custom title' # Should not be overwritten.
|
||||
bar_file_group.title = 'custom title' # Should not be overwritten.
|
||||
bar_file_group.add_tag(tag.id)
|
||||
test_session.commit()
|
||||
|
||||
|
|
@ -945,7 +957,7 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make
|
|||
qux.mkdir()
|
||||
|
||||
# Move both files into qux. The Tag should also be moved.
|
||||
await lib.move(qux, bar, foo)
|
||||
await lib.move(qux, bar, foo, session=test_session)
|
||||
new_foo = qux / 'foo.txt'
|
||||
new_bar = qux / 'bar.txt'
|
||||
# Files were moved.
|
||||
|
|
@ -963,10 +975,10 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make
|
|||
assert not foo_file_group.tag_files
|
||||
assert foo_file_group.primary_path.is_file()
|
||||
assert foo_file_group.primary_path == new_foo == foo_file_group.files[0]['path']
|
||||
assert foo_file_group.title == 'custom title', 'Custom title should not have been overwritten.'
|
||||
assert bar_file_group.title == 'custom title', 'Custom title should not have been overwritten.'
|
||||
|
||||
# Rename "bar.txt" to "baz.txt"
|
||||
await lib.rename(new_bar, 'baz.txt')
|
||||
await lib.rename(new_bar, 'baz.txt', session=test_session)
|
||||
baz = new_bar.with_name('baz.txt')
|
||||
assert baz.read_text() == 'bar'
|
||||
|
||||
|
|
@ -1111,3 +1123,21 @@ async def test_upsert_file_video_with_channel(test_async_client, test_session, t
|
|||
assert video.channel_id == channel.id, "Videos should be assigned to the Channel their files are in"
|
||||
assert video.video_path, 'Video file was upserted'
|
||||
assert video.info_json_path, 'Info json file not found near video file.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_many_files(test_async_client, test_session, test_directory, make_files_structure):
|
||||
make_files_structure([f'foo/{i}.txt' for i in range(10_000)])
|
||||
await lib.refresh_files()
|
||||
|
||||
foo = test_directory / 'foo'
|
||||
bar = test_directory / 'bar'
|
||||
|
||||
with timer('test_move_many_files'):
|
||||
# mv foo bar
|
||||
await lib.move(bar, foo, session=test_session)
|
||||
|
||||
assert not foo.is_dir()
|
||||
assert bar.is_dir()
|
||||
assert (bar / 'foo').is_dir()
|
||||
assert (bar / 'foo/0.txt').is_file()
|
||||
|
|
|
|||
21
wrolpi/files/test/test_models.py
Normal file
21
wrolpi/files/test/test_models.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_move(test_async_client, test_session, test_directory, video_factory):
|
||||
"""Any new or renamed files are adopted during FileGroup.move."""
|
||||
video = video_factory(title='video', with_poster_ext='.webp')
|
||||
test_session.commit()
|
||||
|
||||
# Move one of the FileGroup's files. It should be moved as well.
|
||||
video.poster_path.rename(video.poster_path.with_suffix('.jpg'))
|
||||
|
||||
# Move the FilGroup.
|
||||
new_primary_path = video.video_path.with_name('new name')
|
||||
video.file_group.move(new_primary_path)
|
||||
|
||||
# New file is discovered and adopted, missing file is removed.
|
||||
assert video.file_group.my_paths() == [
|
||||
(test_directory / 'videos/NO CHANNEL/new name.jpg'),
|
||||
(test_directory / 'videos/NO CHANNEL/new name.mp4'),
|
||||
]
|
||||
|
|
@ -23,7 +23,7 @@ from wrolpi.admin import HotspotStatus
|
|||
from wrolpi.api_utils import json_response, json_error_handler, api_app
|
||||
from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \
|
||||
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \
|
||||
set_global_log_level, get_relative_to_media_directory
|
||||
set_global_log_level, get_relative_to_media_directory, search_other_estimates
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
|
||||
|
|
@ -31,6 +31,7 @@ from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
|
|||
from wrolpi.events import get_events, Events
|
||||
from wrolpi.files import files_bp
|
||||
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.vars import DOCKERIZED, IS_RPI, IS_RPI4, IS_RPI5, API_HOST, API_PORT, API_WORKERS, API_DEBUG, \
|
||||
API_ACCESS_LOG, truthy_arg, API_AUTO_RELOAD
|
||||
from wrolpi.version import __version__
|
||||
|
|
@ -118,10 +119,16 @@ async def index(_):
|
|||
@openapi.description('Echo whatever is sent to this.')
|
||||
@openapi.response(HTTPStatus.OK, schema.EchoResponse)
|
||||
async def echo(request: Request):
|
||||
try:
|
||||
request_json = request.json
|
||||
except Exception as e:
|
||||
logger.error('Failed to parse request JSON', exc_info=e)
|
||||
request_json = None
|
||||
request_headers = dict(request.headers)
|
||||
ret = dict(
|
||||
form=request.form,
|
||||
headers=dict(request.headers),
|
||||
json=request.json,
|
||||
headers=request_headers,
|
||||
json=request_json,
|
||||
method=request.method,
|
||||
args=request.args,
|
||||
)
|
||||
|
|
@ -137,7 +144,7 @@ def get_settings(_: Request):
|
|||
ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories]
|
||||
|
||||
settings = {
|
||||
'archive_directory': config.archive_destination,
|
||||
'archive_destination': config.archive_destination,
|
||||
'download_manager_disabled': download_manager.is_disabled,
|
||||
'download_manager_stopped': download_manager.is_stopped,
|
||||
'download_on_startup': config.download_on_startup,
|
||||
|
|
@ -150,15 +157,15 @@ def get_settings(_: Request):
|
|||
'ignore_outdated_zims': config.ignore_outdated_zims,
|
||||
'ignored_directories': ignored_directories,
|
||||
'log_level': api_app.shared_ctx.log_level.value,
|
||||
'map_directory': config.map_destination,
|
||||
'map_destination': config.map_destination,
|
||||
'nav_color': config.nav_color,
|
||||
'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative.
|
||||
'throttle_on_startup': config.throttle_on_startup,
|
||||
'throttle_status': admin.throttle_status().name,
|
||||
'version': __version__,
|
||||
'videos_directory': config.videos_destination,
|
||||
'videos_destination': config.videos_destination,
|
||||
'wrol_mode': config.wrol_mode,
|
||||
'zims_directory': config.zims_destination,
|
||||
'zims_destination': config.zims_destination,
|
||||
}
|
||||
return json_response(settings)
|
||||
|
||||
|
|
@ -482,7 +489,7 @@ async def get_tags_request(_: Request):
|
|||
summary='Create or update a Tag',
|
||||
)
|
||||
async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None):
|
||||
tags.upsert_tag(body.name, body.color, tag_id)
|
||||
await tags.upsert_tag(body.name, body.color, tag_id)
|
||||
if tag_id:
|
||||
return response.empty(HTTPStatus.OK)
|
||||
else:
|
||||
|
|
@ -491,7 +498,7 @@ async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None):
|
|||
|
||||
@api_bp.delete('/tag/<tag_id:int>')
|
||||
async def delete_tag_request(_: Request, tag_id: int):
|
||||
tags.delete_tag(tag_id)
|
||||
Tag.find_by_id(tag_id).delete()
|
||||
return response.empty()
|
||||
|
||||
|
||||
|
|
@ -594,4 +601,13 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate
|
|||
return json_response(ret)
|
||||
|
||||
|
||||
@api_bp.post('/search_other_estimates')
|
||||
@validate(json=schema.SearchOtherEstimateRequest)
|
||||
async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstimateRequest):
|
||||
"""Used by the Global search to suggest FileGroup count to the user."""
|
||||
others = await search_other_estimates(body.tag_names)
|
||||
ret = dict(others=others)
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
api_app.error_handler.add(Exception, json_error_handler)
|
||||
|
|
|
|||
|
|
@ -206,3 +206,8 @@ class SearchFileEstimateRequest:
|
|||
def __post_init__(self):
|
||||
if self.any_tag and self.tag_names:
|
||||
raise ValidationError('Cannot search for any tag, and list of tags.')
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchOtherEstimateRequest:
|
||||
tag_names: List[str] = field(default_factory=lambda: list())
|
||||
|
|
|
|||
|
|
@ -54,7 +54,9 @@ class Tag(ModelHelper, Base):
|
|||
color = Column(String)
|
||||
|
||||
tag_files: List[TagFile] = relationship('TagFile', back_populates='tag', cascade='all')
|
||||
tag_zim_entries: List[TagFile] = relationship('TagZimEntry', back_populates='tag', cascade='all')
|
||||
tag_zim_entries: List = relationship('TagZimEntry', back_populates='tag', cascade='all')
|
||||
channels: List = relationship('Channel', primaryjoin='Tag.id==Channel.tag_id', back_populates='tag',
|
||||
cascade='all')
|
||||
|
||||
def __repr__(self):
|
||||
name = self.name
|
||||
|
|
@ -148,27 +150,71 @@ class Tag(ModelHelper, Base):
|
|||
|
||||
def has_relations(self) -> bool:
|
||||
"""Returns True if this Tag has been used with any FileGroups or Zim Entries."""
|
||||
return bool(self.tag_files or self.tag_zim_entries)
|
||||
return bool(any(self.tag_files) or any(self.tag_zim_entries) or any(self.channels))
|
||||
|
||||
def update_tag(self, name: str, color: str, session: Session):
|
||||
async def update_tag(self, name: str, color: str | None, session: Session):
|
||||
"""Change name/color of tag. Ensures safeness of new values, checks for conflicts."""
|
||||
if ',' in name:
|
||||
raise InvalidTag('Tag name cannot have comma')
|
||||
if not is_valid_hex_color(color):
|
||||
if color and not is_valid_hex_color(color):
|
||||
raise InvalidTag('Tag color is invalid')
|
||||
|
||||
# Replace forward-slash (linux directories) with unicode Big Solidus (U+29F8)
|
||||
# Replace forward-slash (linux directories) with Unicode Big Solidus (U+29F8)
|
||||
name = name.replace('/', '⧸')
|
||||
name = INVALID_FILE_CHARS.sub('', name)
|
||||
|
||||
if Tag.get_by_name(name, session):
|
||||
if (other := self.get_by_name(name, session)) and other.id != self.id:
|
||||
raise InvalidTag('Tag name already taken')
|
||||
|
||||
if flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to rename tag while file refresh is in progress')
|
||||
|
||||
old_name = self.name
|
||||
self.name = name
|
||||
self.color = color
|
||||
self.color = color or self.color
|
||||
if Session.object_session(self):
|
||||
self.flush()
|
||||
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.errors import ChannelDirectoryConflict
|
||||
|
||||
async def _():
|
||||
for channel in self.channels:
|
||||
channel: Channel
|
||||
possible_directory = channel.format_directory(old_name)
|
||||
if channel.directory == possible_directory:
|
||||
# Channel is in this Tag's old directory, move the Channel to the new directory.
|
||||
new_directory = channel.format_directory(name)
|
||||
try:
|
||||
new_directory.mkdir(parents=True)
|
||||
except FileExistsError:
|
||||
raise ChannelDirectoryConflict(f'Channel directory already exists: {new_directory}')
|
||||
|
||||
# Move the files of the Channel.
|
||||
await channel.move_channel(new_directory, session)
|
||||
else:
|
||||
msg = f"Not moving Channel because it is not in this Tag's old directory:" \
|
||||
f" {channel} {possible_directory} {self}"
|
||||
logger.warning(msg)
|
||||
|
||||
if PYTEST:
|
||||
await _()
|
||||
else:
|
||||
background_task(_())
|
||||
|
||||
def delete(self):
|
||||
"""Deletes this Tag, if it is unused.
|
||||
|
||||
@warning: Commits and saves configs"""
|
||||
if self.has_relations():
|
||||
raise UsedTag(f'Cannot delete {self.name} it is used')
|
||||
|
||||
session = Session.object_session(self)
|
||||
session.delete(self)
|
||||
session.commit()
|
||||
|
||||
save_tags_config(session)
|
||||
sync_tags_directory(session)
|
||||
|
||||
|
||||
class TagsConfig(ConfigFile):
|
||||
|
|
@ -394,16 +440,17 @@ def get_tags() -> List[dict]:
|
|||
|
||||
@optional_session
|
||||
@run_after(save_tags_config)
|
||||
def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag:
|
||||
async def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag:
|
||||
if tag_id:
|
||||
tag = Tag.find_by_id(tag_id, session)
|
||||
tag.update_tag(name, color, session)
|
||||
tag.flush()
|
||||
await tag.update_tag(name, color, session)
|
||||
else:
|
||||
tag = Tag()
|
||||
tag.update_tag(name, color, session)
|
||||
await tag.update_tag(name, color, session)
|
||||
session.add(tag)
|
||||
|
||||
tag.flush()
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except IntegrityError as e:
|
||||
|
|
@ -411,26 +458,12 @@ def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = Non
|
|||
session.rollback()
|
||||
raise InvalidTag(f'Name already taken') from e
|
||||
|
||||
save_tags_config()
|
||||
sync_tags_directory()
|
||||
save_tags_config(session)
|
||||
sync_tags_directory(session)
|
||||
|
||||
return tag
|
||||
|
||||
|
||||
@optional_session
|
||||
def delete_tag(tag_id: int, session: Session = None):
|
||||
tag = Tag.find_by_id(tag_id, session)
|
||||
|
||||
if tag.has_relations():
|
||||
raise UsedTag(f'Cannot delete {tag.name} it is used')
|
||||
|
||||
session.delete(tag)
|
||||
session.commit()
|
||||
|
||||
save_tags_config()
|
||||
sync_tags_directory()
|
||||
|
||||
|
||||
@optional_session
|
||||
def import_tags_config(session: Session = None):
|
||||
"""Reads the Tags and TagFiles from the config file, upserts them in the DB."""
|
||||
|
|
|
|||
|
|
@ -901,3 +901,18 @@ def test_html_screenshot(singlefile_contents_factory):
|
|||
])
|
||||
def test_is_valid_hex_color(color, expected):
|
||||
assert common.is_valid_hex_color(color) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('iterable,predicate,expected', [
|
||||
((1, 2, 2, 3, 4), lambda i: i, (1, 2, 3, 4)),
|
||||
([1, 1, 2, 3, 3, 3, 3, 4], None, [1, 2, 3, 4]),
|
||||
([1, 1, 2, 2, 3, 4, 5, 6, 7, 8], lambda i: i % 2, [1, 2]),
|
||||
(['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach'], lambda i: len(i), ['apple', 'banana', 'pear']),
|
||||
(
|
||||
[pathlib.Path('foo.txt'), pathlib.Path('foo.mp4'), pathlib.Path('bar.txt')],
|
||||
lambda i: i.stem,
|
||||
[pathlib.Path('foo.txt'), pathlib.Path('bar.txt')]
|
||||
),
|
||||
])
|
||||
def test_unique_by_predicate(iterable, predicate, expected):
|
||||
assert common.unique_by_predicate(iterable, predicate) == expected
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ async def test_create_downloads(test_async_client, test_session, test_download_m
|
|||
@pytest.mark.asyncio
|
||||
async def test_recreate_download(test_session, test_download_manager, test_downloader, tag_factory):
|
||||
"""Settings are preserved when a Download is restarted."""
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
settings = {'tag_names': [tag.name, ]}
|
||||
|
||||
# Create download with settings initially.
|
||||
|
|
|
|||
|
|
@ -392,7 +392,7 @@ async def test_get_status(test_async_client, test_session):
|
|||
@pytest.mark.asyncio
|
||||
async def test_download_crud(test_session, test_async_client, test_download_manager_config, tag_factory):
|
||||
"""Test creating once-downloads and recurring downloads."""
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
|
||||
async def dispatch_downloads(*a, **kw):
|
||||
pass
|
||||
|
|
@ -567,7 +567,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
|
|||
request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_)
|
||||
assert response_.status_code == HTTPStatus.OK
|
||||
|
||||
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
|
||||
tag1, tag2, tag3 = await tag_factory(), await tag_factory(), await tag_factory()
|
||||
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
||||
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
|
||||
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
|
||||
|
|
@ -629,7 +629,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
|
|||
async def test_search_file_estimates_any_tag(test_session, test_async_client, archive_factory, tag_factory,
|
||||
video_factory):
|
||||
"""File estimates can be filtered by any_tag."""
|
||||
tag1 = tag_factory()
|
||||
tag1 = await tag_factory()
|
||||
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
||||
archive_factory(domain='bar.com', contents='contents of bar bunny')
|
||||
test_session.commit()
|
||||
|
|
@ -706,6 +706,12 @@ def test_recursive_errors():
|
|||
@pytest.mark.asyncio
|
||||
async def test_settings_special_directories(test_async_client, test_config):
|
||||
"""Maintainer can change special directories."""
|
||||
request, response = await test_async_client.get('/api/settings')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['archive_destination'] == 'archive/%(domain)s'
|
||||
assert response.json['map_destination'] == 'map'
|
||||
assert response.json['videos_destination'] == 'videos/%(channel_tag)s/%(channel_name)s'
|
||||
assert response.json['zims_destination'] == 'zims'
|
||||
|
||||
data = {'archive_destination': 'archives'}
|
||||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
|
|
@ -731,3 +737,23 @@ async def test_settings_special_directories(test_async_client, test_config):
|
|||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert get_wrolpi_config().archive_destination == 'archive/%(domain)s'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_other_estimates(test_async_client, test_session, channel_factory, tag_factory):
|
||||
# Can search without Tags or Channels.
|
||||
body = dict(tag_names=[])
|
||||
request, response = await test_async_client.post('/api/search_other_estimates', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['others']['channel_count'] == 0, 'Only one Channel is tagged.'
|
||||
|
||||
tag = await tag_factory()
|
||||
channel1 = channel_factory(tag_name=tag.name)
|
||||
channel2 = channel_factory()
|
||||
test_session.commit()
|
||||
|
||||
# One Channel is tagged.
|
||||
body = dict(tag_names=[tag.name, ])
|
||||
request, response = await test_async_client.post('/api/search_other_estimates', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import yaml
|
|||
|
||||
from wrolpi import tags
|
||||
from wrolpi.common import is_hardlinked, walk
|
||||
from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged
|
||||
from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged, UsedTag
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import TagFile, Tag
|
||||
|
|
@ -15,8 +15,8 @@ from wrolpi.tags import TagFile, Tag
|
|||
@pytest.mark.asyncio
|
||||
async def test_tags_file_group_json(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
|
||||
"""The tags of a file are returned in its JSON."""
|
||||
tag_one = tag_factory()
|
||||
tag_two = tag_factory()
|
||||
tag_one = await tag_factory()
|
||||
tag_two = await tag_factory()
|
||||
await files_lib.refresh_files()
|
||||
file_group: FileGroup = test_session.query(FileGroup).one()
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ async def test_tags_file_group_json(test_async_client, test_session, make_files_
|
|||
@pytest.mark.asyncio
|
||||
async def test_tags_model(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
|
||||
"""Can get Tag using class methods."""
|
||||
tag1 = tag_factory()
|
||||
tag1 = await tag_factory()
|
||||
assert tag1.__json__()
|
||||
|
||||
assert Tag.find_by_id(tag1.id)
|
||||
|
|
@ -63,7 +63,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc
|
|||
await files_lib.refresh_files()
|
||||
video_group: FileGroup = test_session.query(FileGroup).one()
|
||||
|
||||
tag_one = tag_factory()
|
||||
tag_one = await tag_factory()
|
||||
assert len(tag_one.tag_files) == 0
|
||||
|
||||
Tag.tag_file_group(video_group.id, tag_one.id)
|
||||
|
|
@ -71,7 +71,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc
|
|||
assert len(tag_one.tag_files) == 1
|
||||
assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ]
|
||||
|
||||
tag_two = tag_factory()
|
||||
tag_two = await tag_factory()
|
||||
Tag.tag_file_group(video_group.id, tag_two.id)
|
||||
test_session.commit()
|
||||
assert len(tag_one.tag_files) == 1
|
||||
|
|
@ -95,8 +95,8 @@ async def test_tags_config_file(test_async_client, test_session, test_directory,
|
|||
await files_lib.refresh_files()
|
||||
pdf: FileGroup = FileGroup.get_by_path(example_pdf, test_session)
|
||||
video: FileGroup = FileGroup.get_by_path(video_file, test_session)
|
||||
tag1 = tag_factory()
|
||||
tag2 = tag_factory()
|
||||
tag1 = await tag_factory()
|
||||
tag2 = await tag_factory()
|
||||
test_session.commit()
|
||||
|
||||
tags.save_tags_config(test_session)
|
||||
|
|
@ -172,11 +172,17 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta
|
|||
dict(name='ガーデン', color='#000000', id=1, file_group_count=1, zim_entry_count=0)]
|
||||
assert_tags_config(tags={'ガーデン': {'color': '#000000'}})
|
||||
|
||||
# Tag color can be changed
|
||||
content = dict(name='ガーデン', color='#ffffff')
|
||||
request, response = await test_async_client.post('/api/tag/1', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert Tag.find_by_name('ガーデン').color == '#ffffff'
|
||||
|
||||
# Conflicting names return an error.
|
||||
content = dict(name='ガーデン', color='#111111')
|
||||
request, response = await test_async_client.post('/api/tag', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert_tags_config(tags={'ガーデン': {'color': '#000000'}})
|
||||
assert_tags_config(tags={'ガーデン': {'color': '#ffffff'}})
|
||||
|
||||
# Cannot delete Tag that is used.
|
||||
request, response = await test_async_client.delete('/api/tag/1')
|
||||
|
|
@ -197,7 +203,7 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta
|
|||
async def test_delete_tagged_file(test_session, example_pdf, tag_factory):
|
||||
"""You cannot delete a FileGroup if it is tagged."""
|
||||
await files_lib.refresh_files()
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
|
||||
pdf: FileGroup = test_session.query(FileGroup).one()
|
||||
pdf.add_tag(tag.id)
|
||||
|
|
@ -277,7 +283,7 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
|
|||
from modules.zim import lib as zim_lib
|
||||
|
||||
# Create some tags, use tag1.
|
||||
tag1, tag2, tag3, tag4 = tag_factory(), tag_factory(), tag_factory(), tag_factory()
|
||||
tag1, tag2, tag3, tag4 = await tag_factory(), await tag_factory(), await tag_factory(), await tag_factory()
|
||||
# use tag1 for FileGroup.
|
||||
foo, = make_files_structure({'foo': 'text'})
|
||||
foo = FileGroup.from_paths(test_session, foo)
|
||||
|
|
@ -324,16 +330,13 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
|
|||
@pytest.mark.asyncio
|
||||
async def test_invalid_tag(test_async_client, test_session, test_directory):
|
||||
with pytest.raises(InvalidTag):
|
||||
tags.upsert_tag('cannot use comma: ,', '#fff')
|
||||
await tags.upsert_tag('cannot use comma: ,', '#fff')
|
||||
|
||||
with pytest.raises(InvalidTag):
|
||||
tags.upsert_tag('invalid color', 'foo')
|
||||
await tags.upsert_tag('invalid color', 'foo')
|
||||
|
||||
with pytest.raises(UnknownTag):
|
||||
tags.upsert_tag('Tag ID does not exist', '#fff', 1)
|
||||
|
||||
with pytest.raises(UnknownTag):
|
||||
tags.delete_tag(1)
|
||||
await tags.upsert_tag('Tag ID does not exist', '#fff', 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -347,7 +350,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
|
|||
(test_directory / 'tags/cannot be deleted').mkdir(parents=True)
|
||||
(test_directory / 'tags/cannot be deleted/because of this file').touch()
|
||||
|
||||
tag1, tag2 = tag_factory('First Aid'), tag_factory('Special/name')
|
||||
tag1, tag2 = await tag_factory('First Aid'), await tag_factory('Special/name')
|
||||
vid1 = video_factory(with_video_file=test_directory / 'vid1.mp4', with_caption_file=True, with_info_json=True,
|
||||
with_poster_ext='jpg')
|
||||
vid2 = video_factory(with_video_file=test_directory / 'vid2.mp4', with_info_json=True)
|
||||
|
|
@ -443,7 +446,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
|
|||
async def test_update_tag(test_async_client, test_session, test_directory, video_factory, tag_factory):
|
||||
"""Linked files in the Tag Directory are moved when the tag is renamed."""
|
||||
# Create tagged Video.
|
||||
tag = tag_factory()
|
||||
tag = await tag_factory()
|
||||
video_factory(title='video', tag_names=[tag.name])
|
||||
test_session.commit()
|
||||
|
||||
|
|
@ -453,10 +456,33 @@ async def test_update_tag(test_async_client, test_session, test_directory, video
|
|||
assert (test_directory / 'tags/one/video.mp4').is_file()
|
||||
|
||||
# Invalid characters are replaced during rename.
|
||||
tags.upsert_tag('new/name%', tag.color, tag.id)
|
||||
await tags.upsert_tag('new/name%', tag.color, tag.id)
|
||||
|
||||
# Video file was moved.
|
||||
assert (test_directory / 'tags').is_dir()
|
||||
assert (test_directory / 'videos/NO CHANNEL/video.mp4').is_file()
|
||||
assert not (test_directory / 'tags/one/video.mp4').exists()
|
||||
assert (test_directory / 'tags/new⧸name/video.mp4').is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_rename_with_channel(test_async_client, test_session, test_directory, video_factory, tag_factory,
|
||||
channel_factory):
|
||||
tag = await tag_factory()
|
||||
channel = channel_factory(name='Channel Name', tag_name=tag.name)
|
||||
video_factory(title='video', channel_id=channel.id)
|
||||
test_session.commit()
|
||||
|
||||
assert (test_directory / 'tags').is_dir()
|
||||
assert (test_directory / 'videos/one/Channel Name').is_dir()
|
||||
assert (test_directory / 'videos/one/Channel Name/video.mp4').is_file()
|
||||
|
||||
await tag.update_tag('New Tag Name', None, session=test_session)
|
||||
|
||||
assert (test_directory / 'tags').is_dir()
|
||||
assert (test_directory / 'videos/New Tag Name/Channel Name').is_dir()
|
||||
assert (test_directory / 'videos/New Tag Name/Channel Name/video.mp4').is_file()
|
||||
|
||||
# Cannot delete Tag when used by Channel. This prevents the need to move Channel directories when Tags are deleted.
|
||||
with pytest.raises(UsedTag):
|
||||
tag.delete()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue