Moving FileGroups instead of refreshing files after they have been moved.

This commit is contained in:
Roland 2024-08-15 19:32:01 -06:00
parent 6fe8a7b704
commit 0f232fd533
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
59 changed files with 1902 additions and 1040 deletions

View file

@ -75,6 +75,14 @@ function handleEvents(events) {
eventToast('Successful tag', message, 'success');
}
if (event === 'file_move_completed') {
eventToast('Successful Move', message, 'success');
}
if (event === 'file_move_failed') {
eventToast('Move Failed', message, 'error');
}
if (subject) {
newestEvents[subject] = dt;
}

File diff suppressed because it is too large Load diff

View file

@ -118,13 +118,6 @@ function ArchivePage() {
description: 'Archive update has been scheduled.',
time: 2000,
});
} else {
toast({
type: 'error',
title: 'Archive Downloading',
description: 'Archive update failed.',
time: 2000,
});
}
}

View file

@ -1,5 +1,5 @@
import React, {useState} from "react";
import {Dimmer, Dropdown, Grid, Label, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
import {Dimmer, Dropdown, Grid, Input, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
import {
createChannel,
createChannelDownload,
@ -19,6 +19,7 @@ import {
frequencyOptions,
HelpHeader,
humanFileSize,
humanNumber,
RequiredAsterisk,
SearchInput,
secondsToFrequency,
@ -77,6 +78,10 @@ function ChannelStatistics({statistics}) {
<StatisticValue>{secondsToFullDuration(statistics.length)}</StatisticValue>
<StatisticLabel>Total Duration</StatisticLabel>
</Statistic>
<Statistic>
<StatisticValue>{humanNumber(statistics.video_tags)}</StatisticValue>
<StatisticLabel>Video Tags</StatisticLabel>
</Statistic>
</Segment>
}
@ -231,15 +236,7 @@ function ChannelPage({create, header}) {
if (e) {
e.preventDefault();
}
const response = await refreshChannel(channelId);
if (response.status !== 204) {
toast({
type: 'error',
title: 'Failed to refresh',
description: "Failed to refresh this channel's directory",
time: 5000,
})
}
await refreshChannel(channelId);
}
const handleDelete = async () => {
@ -295,19 +292,36 @@ function ChannelPage({create, header}) {
>
<ModalHeader>{channel.tag_name ? 'Modify Tag' : 'Add Tag'}</ModalHeader>
<ModalContent>
<TagsSelector
limit={1}
selectedTagNames={newTagName ? [newTagName] : []}
onAdd={handleTagSelect}
onRemove={() => handleTagSelect(null)}
/>
<br/>
<Toggle
label='Move to directory: '
checked={moveToTagDirectory}
onChange={setMoveToTagDirectory}
/>
{newTagDirectory && <Label size='large'>{newTagDirectory}</Label>}
<Grid columns={1}>
<Grid.Row>
<Grid.Column>
<TagsSelector
limit={1}
selectedTagNames={newTagName ? [newTagName] : []}
onAdd={handleTagSelect}
onRemove={() => handleTagSelect(null)}
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<Toggle
label='Move to directory: '
checked={moveToTagDirectory}
onChange={setMoveToTagDirectory}
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<Input fluid
value={newTagDirectory}
onChange={(e, {value}) => setNewTagDirectory(value)}
disabled={!moveToTagDirectory}
/>
</Grid.Column>
</Grid.Row>
</Grid>
</ModalContent>
<ModalActions>
<Button onClick={() => setTagModalOpen(false)}>Cancel</Button>
@ -354,7 +368,6 @@ function ChannelPage({create, header}) {
Directory <RequiredAsterisk/>
</label>
<DirectoryInput required
disabled={create === undefined}
value={channel.directory}
setInput={value => changeValue('directory', value)}
placeholder='videos/channel directory'
@ -756,16 +769,21 @@ export function ChannelsPage() {
</div>;
const headers = [
{key: 'name', text: 'Name', sortBy: 'name', width: 8},
{key: 'tag', text: 'Tag', sortBy: 'tag_name', width: 2},
{key: 'video_count', text: 'Videos', sortBy: 'video_count', width: 2},
{key: 'download_frequency', text: 'Download Frequency', sortBy: 'minimum_frequency', width: 2},
{key: 'size', text: 'Size', sortBy: 'size', width: 2},
{key: 'name', text: 'Name', sortBy: [i => i['name'].toLowerCase()], width: 8},
{key: 'tag', text: 'Tag', sortBy: [i => i['tag_name'], i => i['name'].toLowerCase()], width: 2},
{key: 'video_count', text: 'Videos', sortBy: [i => i['video_count'], i => i['name'].toLowerCase()], width: 2},
{
key: 'download_frequency',
text: 'Download Frequency',
sortBy: [i => i['minimum_frequency'], i => i['name'].toLowerCase()],
width: 2
},
{key: 'size', text: 'Size', sortBy: [i => i['size'], i => i['name'].toLowerCase()], width: 2},
{key: 'manage', text: 'Manage', width: 2},
];
const mobileHeaders = [
{key: 'name', text: 'Name', sortBy: 'name'},
{key: 'video_count', text: 'Videos', sortBy: 'video_count'},
{key: 'name', text: 'Name', sortBy: [i => i['name'].toLowerCase()]},
{key: 'video_count', text: 'Videos', sortBy: [i => i['video_count'], i => i['name'].toLowerCase()]},
{key: 'manage', text: 'Manage'},
];

View file

@ -64,6 +64,7 @@ export const DEFAULT_LIMIT = 20;
export const NAME = process.env && process.env.REACT_APP_NAME ? process.env.REACT_APP_NAME : null;
// Other services on a WROLPi.
export const FILES_MEDIA_URI = `https://${window.location.hostname}:${window.location.port}/media/`;
export const MAP_VIEWER_URI = `https://${window.location.hostname}:8084`;
export const ZIM_VIEWER_URI = `https://${window.location.hostname}:8085`;
export const HELP_VIEWER_URI = `https://${window.location.hostname}:8086`;
@ -1281,7 +1282,10 @@ export function DirectoryInput({disabled, error, placeholder, setInput, value, r
setDirectory(value);
}
const {media_directory} = settings;
let {media_directory} = settings;
if (media_directory && !media_directory.endsWith('/')) {
media_directory = `${media_directory}/`;
}
error = error || isDirectory && !isDir;
return <div>

View file

@ -43,10 +43,8 @@ class Downloader extends React.Component {
this.setState({pending: true, submitted: false});
try {
let response = await postDownload(urls, downloader, null, null, null, destination, tagNames);
if (response.status === 204) {
if (response.ok) {
this.setState({urls: '', pending: false, submitted: true});
} else {
toast({type: 'error', title: 'Error', description: 'Failed to create download!'});
}
} finally {
this.setState({pending: false});
@ -216,8 +214,7 @@ class ChannelDownload extends React.Component {
if (response.status === 204) {
this.setState({pending: false, disabled: false, success: true, url: '', ready: false});
} else {
const content = await response.json();
let error = content.message || null;
let error = (await response.json()).message || null;
this.setState({pending: false, disabled: false, success: false, error});
}
}
@ -355,8 +352,7 @@ class RSSDownload extends ChannelDownload {
if (response.status === 204) {
this.setState({pending: false, disabled: false, success: true, url: '', ready: false});
} else {
const content = await response.json();
let error = content.message || null;
let error = (await response.json()).message || null;
this.setState({pending: false, disabled: false, success: false, error});
}
}
@ -487,15 +483,8 @@ class ScrapeDownloader extends Downloader {
suffix,
max_pages,
);
if (response.status === 204) {
if (response.ok) {
this.setState({urls: '', pending: false, submitted: true});
} else {
const content = await response.json();
let error = 'Failed to create download!';
if (content && content['error']) {
error = content['error'];
}
toast({type: 'error', title: 'Error', description: error});
}
} finally {
this.setState({pending: false});

View file

@ -20,7 +20,9 @@ import {
ErrorMessage,
ExternalCardLink,
FileIcon,
FILES_MEDIA_URI,
findPosterPath,
HandPointMessage,
humanFileSize,
isoDatetimeToString,
mimetypeColor,
@ -583,6 +585,10 @@ function FilesPage() {
return <>
<FilesRefreshProgress/>
<FileBrowser/>
<HandPointMessage>
<p>You can also view your media directory at <a href={FILES_MEDIA_URI}>{FILES_MEDIA_URI}</a></p>
</HandPointMessage>
</>;
}

View file

@ -504,35 +504,19 @@ class EditInventory extends React.Component {
handleSubmit = async (inventory, closeCallback) => {
let inventoryId = inventory.id;
delete inventory['id'];
let response = await updateInventory(inventoryId, inventory);
if (response.status === 204) {
if (response.ok) {
await this.props.success();
closeCallback();
} else {
toast({
type: 'error',
header: 'Failed to save inventory!',
description: 'Server responded with an error.',
time: 5000,
})
}
}
handleDelete = async (inventory, closeCallback) => {
let response = await deleteInventory(inventory.id);
if (response.status === 204) {
if (response.ok) {
await this.props.success();
closeCallback();
} else {
toast({
type: 'error',
header: 'Failed to delete inventory!',
description: 'Server responded with an error.',
time: 5000,
})
}
}
@ -554,16 +538,9 @@ class NewInventory extends React.Component {
delete inventory.id;
let response = await saveInventory(inventory);
if (response.status === 201) {
if (response.ok) {
await this.props.success();
closeCallback();
} else {
toast({
type: 'error',
header: 'Failed to save inventory!',
description: 'Server responded with an error.',
time: 5000,
})
}
}
@ -953,18 +930,11 @@ class InventoryAddList extends React.Component {
expiration_date: this.state.expiration_date,
};
let response = await saveItem(this.state.inventory.id, item);
if (response.status === 204) {
if (response.ok) {
await this.clearInputs();
await this.fetchItems();
await this.fetchCategories();
await this.fetchBrands();
} else {
toast({
type: 'error',
header: 'Failed to save item!',
description: 'Server responded with an error.',
time: 5000,
})
}
}

View file

@ -1,15 +1,24 @@
import React, {useState} from "react";
import {Route, Routes, useNavigate} from "react-router-dom";
import {Link, Route, Routes, useNavigate} from "react-router-dom";
import {FilesSearchView} from "./Files";
import {useLatestRequest, usePages, useSearchDate, useSearchFilter} from "../hooks/customHooks";
import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks";
import {ZimSearchView} from "./Zim";
import {searchEstimateFiles, searchEstimateZims, searchSuggestions} from "../api";
import {searchEstimateFiles, searchEstimateOthers, searchEstimateZims, searchSuggestions} from "../api";
import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
import _ from "lodash";
import {TagsContext} from "../Tags";
import {Button as SButton, Header as SHeader, Label} from "semantic-ui-react";
import {Modal, ModalContent} from "./Theme";
import {QueryContext} from "../contexts/contexts";
import {
AccordionContent,
AccordionTitle,
Button as SButton,
Grid,
GridColumn,
GridRow,
Header as SHeader,
Label
} from "semantic-ui-react";
import {Accordion, Header, Icon, Loader, Modal, ModalContent, Segment} from "./Theme";
import {QueryContext, ThemeContext} from "../contexts/contexts";
const SUGGESTED_APPS = [
{location: '/admin', title: 'Downloads', description: 'View and control your downloads'},
@ -128,6 +137,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
const {data: filesData, sendRequest: sendFilesRequest, loading: filesLoading} = useLatestRequest(500);
// Zims are slow, so they are separate.
const {data: zimData, sendRequest: sendZimRequest, loading: zimLoading} = useLatestRequest(500);
const {data: otherData, sendRequest: sendOtherRequest, loading: otherLoading} = useLatestRequest(500);
React.useEffect(() => {
if (searchStr || (tagNames && tagNames.length > 0)) {
@ -139,6 +149,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
}
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
sendOtherRequest(async () => await searchEstimateOthers(tagNames));
}
}, [
searchStr,
@ -179,6 +190,14 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
}
}, [setSuggestions, zimData]);
React.useEffect(() => {
if (!_.isEmpty(otherData)) {
setSuggestions((prevState) => {
return {...prevState, otherEstimates: otherData.others}
});
}
}, [setSuggestions, otherData]);
return {suggestions, loading: generalLoading || zimLoading || filesLoading}
}
@ -269,6 +288,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
}
}
const otherSum = suggestions.otherEstimates ? _.sum(Object.values(suggestions.otherEstimates)) : 0;
const matchingApps = SUGGESTED_APPS.filter(i =>
i.title.toLowerCase().includes(lowerSearchStr)
|| fuzzyMatch(i.title.toLowerCase(), lowerSearchStr)
@ -283,6 +304,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
setSuggestionsSums({
fileGroups: newSuggestions.fileGroups,
zims: zimSum,
otherSum: otherSum,
channels: newSuggestions.channels.length,
domains: newSuggestions.domains.length,
tags: matchingTags?.length,
@ -342,10 +364,12 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
const filesTabName = <span>Files <Label>{normalizeEstimate(suggestionsSums?.fileGroups)}</Label></span>;
const zimsTabName = <span>Zims <Label>{normalizeEstimate(suggestionsSums?.zims)}</Label></span>;
const othersTabName = <span>Other <Label>{normalizeEstimate(suggestionsSums?.otherSum)}</Label></span>;
const links = [
{text: filesTabName, to: '/search', key: 'filesSearch_', end: true},
{text: zimsTabName, to: '/search/zim', key: 'zimsSearch'},
{text: othersTabName, to: '/search/other', key: 'othersSearch'},
];
return <React.Fragment>
@ -353,6 +377,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
<Routes>
<Route path='/*' element={<FilesSearchView/>}/>
<Route path='/zim' exact element={<ZimSearchView suggestions={suggestions} loading={loading}/>}/>
<Route path='/other' exact element={<OtherSearchView loading={loading}/>}/>
</Routes>
</React.Fragment>
}
@ -403,3 +428,58 @@ export function SearchIconButton() {
</Modal>
</React.Fragment>
}
function SearchChannelPreview({channel}) {
const {t} = React.useContext(ThemeContext);
return <GridRow {...t}>
<GridColumn>
<Link to={`/videos/channel/${channel.id}/video`}>
{channel.name}
</Link>
</GridColumn>
</GridRow>
}
function OtherSearchView({loading}) {
const {searchParams} = React.useContext(QueryContext);
const [activeIndex, setActiveIndex] = React.useState(0);
const activeTags = searchParams.getAll('tag');
const {channels, loading: channelsLoading} = useSearchChannels(activeTags);
const handleClick = (newIndex) => {
console.log('newIndex', newIndex, 'activeIndex', activeIndex);
setActiveIndex(activeIndex === newIndex ? null : newIndex);
}
if (loading || channelsLoading) {
return <Accordion>
<Segment><Loader active/></Segment>
</Accordion>
}
const channelsAccordion = <React.Fragment>
<AccordionTitle
index={0}
active={activeIndex === 0}
onClick={() => handleClick(0)}
>
<Header as='h3'>
<Icon name='dropdown'/>
Channels
<Label>{normalizeEstimate(channels.length)}</Label>
</Header>
</AccordionTitle>
<AccordionContent active={activeIndex === 0}>
<Grid>
{!_.isEmpty(channels) ?
channels.map(i => <SearchChannelPreview channel={i}/>)
: 'No Channels'}
</Grid>
</AccordionContent>
</React.Fragment>;
return <Accordion>
{channelsAccordion}
</Accordion>
}

View file

@ -243,6 +243,7 @@ function VideosStatistics() {
{key: 'month', label: 'Downloads Past Month'},
{key: 'year', label: 'Downloads Past Year'},
{key: 'sum_duration', label: 'Total Duration'},
{key: 'censored_videos', label: 'Censored Videos'},
];
const commentsNames = [
{key: 'have_comments', label: 'Have Comments'},
@ -255,6 +256,7 @@ function VideosStatistics() {
];
const channelNames = [
{key: 'channels', label: 'Channels'},
{key: 'tagged_channels', label: 'Tagged Channels'},
];
const buildSegment = (title, names, stats) => {

View file

@ -255,18 +255,7 @@ class RecurringDownloadRow extends React.Component {
handleRestart = async () => {
const {id} = this.props;
try {
const response = await restartDownload(id);
if (response.status !== 204) {
throw Error('Unable to restart download');
}
} catch (e) {
toast({
type: 'error',
title: 'Error',
description: 'Unable to restart download',
time: 5000,
})
throw e;
await restartDownload(id);
} finally {
if (this.props.fetchDownloads) {
this.props.fetchDownloads();

View file

@ -181,7 +181,7 @@ export class SettingsPage extends React.Component {
qrOpen: false,
ready: false,
archive_directory: null,
archive_destination: null,
download_on_startup: null,
download_timeout: null,
hotspot_device: null,
@ -190,12 +190,12 @@ export class SettingsPage extends React.Component {
hotspot_status: null,
ignore_outdated_zims: null,
log_level: null,
map_directory: null,
map_destination: null,
navColor: null,
throttle_on_startup: null,
throttle_status: null,
videos_directory: null,
zims_directory: null,
videos_destination: null,
zims_destination: null,
}
this.handleSubmit = this.handleSubmit.bind(this);
@ -207,7 +207,7 @@ export class SettingsPage extends React.Component {
const settings = await getSettings();
this.setState({
ready: true,
archive_directory: settings.archive_directory,
archive_destination: settings.archive_destination,
disabled: settings.wrol_mode,
download_on_startup: settings.download_on_startup,
download_timeout: settings.download_timeout || '',
@ -217,12 +217,12 @@ export class SettingsPage extends React.Component {
hotspot_status: settings.hotspot_status,
ignore_outdated_zims: settings.ignore_outdated_zims,
log_level: fromApiLogLevel(settings.log_level),
map_directory: settings.map_directory,
map_destination: settings.map_destination,
navColor: settings.nav_color || 'violet',
throttle_on_startup: settings.throttle_on_startup,
throttle_status: settings.throttle_status,
videos_directory: settings.videos_directory,
zims_directory: settings.zims_directory,
videos_destination: settings.videos_destination,
zims_destination: settings.zims_destination,
}, this.handleHotspotChange);
} catch (e) {
console.error(e);
@ -233,7 +233,7 @@ export class SettingsPage extends React.Component {
async handleSubmit(callback) {
this.setState({disabled: true, pending: true});
let settings = {
archive_directory: this.state.archive_directory,
archive_destination: this.state.archive_destination,
download_on_startup: this.state.download_on_startup,
download_timeout: this.state.download_timeout ? parseInt(this.state.download_timeout) : 0,
hotspot_device: this.state.hotspot_device,
@ -241,11 +241,11 @@ export class SettingsPage extends React.Component {
hotspot_ssid: this.state.hotspot_ssid,
ignore_outdated_zims: this.state.ignore_outdated_zims,
log_level: toApiLogLevel(this.state.log_level),
map_directory: this.state.map_directory,
map_destination: this.state.map_destination,
nav_color: this.state.navColor,
throttle_on_startup: this.state.throttle_on_startup,
videos_directory: this.state.videos_directory,
zims_directory: this.state.zims_directory,
videos_destination: this.state.videos_destination,
zims_destination: this.state.zims_destination,
}
try {
const response = await saveSettings(settings);
@ -304,7 +304,7 @@ export class SettingsPage extends React.Component {
}
const {
archive_directory,
archive_destination,
disabled,
download_on_startup,
download_timeout,
@ -314,13 +314,13 @@ export class SettingsPage extends React.Component {
hotspot_ssid,
ignore_outdated_zims,
log_level,
map_directory,
map_destination,
navColor,
pending,
qrCodeValue,
throttle_on_startup,
videos_directory,
zims_directory,
videos_destination,
zims_destination,
} = this.state;
const qrButton = <Button icon style={{marginBottom: '1em'}}><Icon name='qrcode' size='big'/></Button>;
@ -468,9 +468,9 @@ export class SettingsPage extends React.Component {
<label>Archive Directory</label>
<Input
label={mediaDirectoryLabel}
value={archive_directory}
value={archive_destination}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'archive_directory', d.value)}
onChange={(e, d) => this.handleInputChange(e, 'archive_destination', d.value)}
/>
</FormField>
</GridColumn>
@ -479,9 +479,9 @@ export class SettingsPage extends React.Component {
<label>Videos Directory</label>
<Input
label={mediaDirectoryLabel}
value={videos_directory}
value={videos_destination}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'videos_directory', d.value)}
onChange={(e, d) => this.handleInputChange(e, 'videos_destination', d.value)}
/>
</FormField>
</GridColumn>
@ -492,9 +492,9 @@ export class SettingsPage extends React.Component {
<label>Map Directory</label>
<Input
label={mediaDirectoryLabel}
value={map_directory}
value={map_destination}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'map_directory', d.value)}
onChange={(e, d) => this.handleInputChange(e, 'map_destination', d.value)}
/>
</FormField>
</GridColumn>
@ -503,9 +503,9 @@ export class SettingsPage extends React.Component {
<label>Zims Directory</label>
<Input
label={mediaDirectoryLabel}
value={zims_directory}
value={zims_destination}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'zims_directory', d.value)}
onChange={(e, d) => this.handleInputChange(e, 'zims_destination', d.value)}
/>
</FormField>
</GridColumn>

View file

@ -17,7 +17,7 @@ import {
getStatus,
getVideo,
getVideosStatistics,
searchArchives,
searchArchives, searchChannels,
searchDirectories,
searchVideos,
searchZim,
@ -1230,4 +1230,31 @@ export const useDockerized = () => {
export const useCalcQuery = () => {
const [calc, setCalc] = useOneQuery('calc');
return {calc, setCalc}
}
export const useSearchChannels = (defaultTagNames) => {
const [tagNames, setTagNames] = useState(defaultTagNames || []);
const [channels, setChannels] = useState([]);
const [loading, setLoading] = useState(false);
const localSearchChannels = async () => {
setLoading(true);
try {
const {channels: newChannels} = await searchChannels(tagNames);
setChannels(newChannels);
} finally {
setLoading(false);
}
};
useEffect(() => {
localSearchChannels();
}, [tagNames]);
return {
tagNames,
setTagNames,
channels,
loading,
}
}

View file

@ -49,17 +49,20 @@ class ArchiveDownloader(Downloader, ABC):
singlefile, readability, screenshot = await self.do_archive(download)
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
archive_id = archive.id
if download.settings and (tag_names := download.settings.get('tag_names')):
for name in tag_names:
archive.add_tag(name)
with get_db_session() as session:
archive = Archive.find_by_id(archive_id, session)
if download.settings and (tag_names := download.settings.get('tag_names')):
for name in tag_names:
archive.add_tag(name, session)
if session := Session.object_session(archive):
session.commit()
if session := Session.object_session(archive):
session.commit()
logger.info(f'Successfully downloaded Archive {download.url} {archive}')
logger.info(f'Successfully downloaded Archive {download.url} {archive}')
return DownloadResult(success=True, location=f'/archive/{archive.id}')
return DownloadResult(success=True, location=f'/archive/{archive.id}')
@optional_session
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:

View file

@ -82,7 +82,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
archive.validate()
for tag_name in tag_names:
archive.add_tag(tag_name)
archive.add_tag(tag_name, test_session)
return archive

View file

@ -205,7 +205,8 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
archive.file_group.download_datetime = now()
archive.url = url
archive.domain = get_or_create_domain(session, url)
session.flush()
archive.flush()
archive.domain.flush()
return archive

View file

@ -315,8 +315,8 @@ class Archive(Base, ModelHelper):
archive = model_archive(file_group, session)
return archive
def add_tag(self, tag_id_or_name: int | str) -> TagFile:
return self.file_group.add_tag(tag_id_or_name)
def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile:
return self.file_group.add_tag(tag_id_or_name, session)
class Domain(Base, ModelHelper):

View file

@ -84,7 +84,7 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_
@pytest.mark.asyncio
async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory):
"""Tagged Archives can be searched."""
tag = tag_factory()
tag = await tag_factory()
archive_factory(domain='example.com', tag_names=[tag.name, ])
test_session.commit()

View file

@ -2,12 +2,15 @@ from http import HTTPStatus
from sanic import Blueprint
from sanic.request import Request
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger
from . import lib, schema
from .channel.api import channel_bp
from .lib import format_videos_destination
from .models import Channel
from .video.api import video_bp
content_bp = Blueprint('VideoContent', '/api/videos')
@ -25,3 +28,20 @@ logger = logger.getChild(__name__)
async def statistics(_: Request):
ret = await lib.get_statistics()
return json_response(ret, HTTPStatus.OK)
@content_bp.post('/tag_info')
@openapi.definition(
description='Get data about tagging a Channel',
body=schema.ChannelTagInfoRequest,
)
@validate(schema.ChannelTagInfoRequest)
async def channel_tag_info(_: Request, body: schema.ChannelTagInfoRequest):
from wrolpi.tags import Tag
channel = Channel.find_by_id(body.channel_id) if body.channel_id else None
channel_name = channel.name if channel else None
channel_url = channel.url if channel else None
tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
videos_destination = format_videos_destination(channel_name, tag_name, channel_url)
ret = dict(videos_destination=videos_destination)
return json_response(ret)

View file

@ -17,7 +17,7 @@ from wrolpi.vars import PYTEST
from . import lib
from .. import schema
from ..errors import UnknownChannel
from ..lib import save_channels_config, format_videos_destination
from ..lib import save_channels_config
from ..models import Channel
channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels')
@ -78,9 +78,9 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
@validate(schema.ChannelPutRequest)
@run_after(save_channels_config)
@wrol_mode_check
def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
async def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
body.directory = pathlib.Path(body.directory) if body.directory else None
channel = lib.update_channel(data=body, channel_id=channel_id)
channel = await lib.update_channel(data=body, channel_id=channel_id)
return response.raw('', HTTPStatus.NO_CONTENT,
headers={'Location': f'/api/videos/channels/{channel.id}'})
@ -138,32 +138,14 @@ async def channel_download(_: Request, channel_id: int, body: schema.ChannelDown
return response.empty()
@channel_bp.post('/<channel_id:int>/tag_info')
@openapi.definition(
description='Get Channel Tag Info',
body=schema.ChannelTagInfoRequest,
)
@validate(schema.ChannelTagInfoRequest)
@run_after(save_channels_config)
@wrol_mode_check
async def channel_tag_info(_: Request, channel_id: int, body: schema.ChannelTagInfoRequest):
from wrolpi.tags import Tag
channel_name = Channel.find_by_id(channel_id).name
tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
videos_destination = format_videos_destination(channel_name, tag_name)
ret = dict(videos_destination=videos_destination)
return json_response(ret)
@channel_bp.post('/<channel_id:int>/tag')
@openapi.definition(
description='Tag Channel',
description='Tag/untag a Channel with a single tag',
body=schema.ChannelTagRequest,
)
@openapi.response(HTTPStatus.NO_CONTENT)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@validate(schema.ChannelTagRequest)
@run_after(save_channels_config)
@wrol_mode_check
async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagRequest):
directory = None
@ -174,3 +156,17 @@ async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagReques
await lib.tag_channel(body.tag_name, directory, channel_id)
return response.raw('', HTTPStatus.NO_CONTENT)
@channel_bp.post('/search')
@openapi.definition(
description='Search Channels',
body=schema.ChannelSearchRequest,
)
@validate(schema.ChannelSearchRequest)
async def channel_search(_: Request, body: schema.ChannelSearchRequest):
channels = await lib.search_channels(body.tag_names)
ret = dict(
channels=channels,
)
return json_response(ret)

View file

@ -7,10 +7,11 @@ from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.common import run_after, logger, \
get_media_directory, wrol_mode_check
get_media_directory, wrol_mode_check, background_task
from wrolpi.db import get_db_curs, optional_session, get_db_session
from wrolpi.downloader import save_downloads_config, download_manager
from wrolpi.errors import UnknownDirectory, APIError, ValidationError, RefreshConflict
from wrolpi.errors import APIError, ValidationError, RefreshConflict
from wrolpi.vars import PYTEST
from .. import schema
from ..common import check_for_channel_conflicts
from ..errors import UnknownChannel, ChannelDirectoryConflict
@ -115,22 +116,10 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
@run_after(save_channels_config)
@optional_session
def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
async def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
"""Update a Channel's DB record"""
channel = Channel.find_by_id(channel_id, session)
# Only update directory if it was empty
if data.directory and not channel.directory:
data.directory = get_media_directory() / data.directory
if not data.directory.is_dir():
if data.mkdir:
data.directory.mkdir()
else:
raise UnknownDirectory()
elif data.directory:
# Keep the old directory because the user can't change a Channel's directory.
data.directory = channel.directory
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
@ -140,11 +129,27 @@ def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_
directory=data.directory,
)
data = data.__dict__
old_directory = pathlib.Path(channel.directory)
new_directory = get_media_directory() / data.pop('directory')
# Apply the changes now that we've OK'd them
channel.update(data.__dict__)
channel.update(data)
session.commit()
async def _():
await channel.move_channel(new_directory, session, send_events=True)
if new_directory != old_directory:
if not new_directory.is_dir():
new_directory.mkdir(parents=True)
if PYTEST:
await _()
else:
background_task(_())
return channel
@ -252,7 +257,7 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
@wrol_mode_check
@optional_session
async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: int, session: Session = None):
async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, channel_id: int, session: Session = None):
"""Add a Tag to a Channel, or remove a Tag from a Channel if no `tag_name` is provided.
Move the Channel to the new directory, if provided."""
@ -261,29 +266,39 @@ async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id:
if directory and flags.refreshing.is_set():
raise RefreshConflict('Refusing to move channel while file refresh is in progress')
channel = Channel.find_by_id(channel_id, session=session)
channel = Channel.find_by_id(channel_id, session)
if tag_name:
tag = Tag.find_by_name(tag_name, session=session)
tag = Tag.find_by_name(tag_name, session)
channel.tag_id = tag.id
channel.tag = tag
else:
channel.tag = channel.tag_id = None
session.flush([channel, ])
channel.flush()
session.commit()
# Do not move Channel files into a directory with files.
if directory and directory.is_dir() and next(directory.iterdir(), None):
if directory and directory != channel.directory and directory.is_dir() and next(directory.iterdir(), None):
raise ChannelDirectoryConflict('Channel directory already exists and is not empty')
# Only move Channel when requested.
if directory:
# Move to newly defined directory only if necessary.
logger.info(f'Tagging {channel} with {tag_name} moving to {directory} from {channel.directory}')
directory.mkdir(parents=True, exist_ok=True)
if channel.directory != directory:
await channel.move(directory, session)
coro = channel.move_channel(directory, session, send_events=True)
if PYTEST:
await coro
else:
background_task(coro)
else:
logger.info(f'Tagging {channel} with {tag_name}')
session.commit()
@optional_session
async def search_channels(tag_names: List[str], session: Session) -> List[Channel]:
"""Search Tagged Channels."""
from wrolpi.tags import Tag
channels = session.query(Channel).join(Tag).filter(Tag.name.in_(tag_names)).all()
return channels

View file

@ -1,6 +1,7 @@
import asyncio
import json
import pathlib
import shutil
import tempfile
from datetime import timedelta
from http import HTTPStatus
@ -96,7 +97,8 @@ def test_video_file_name(test_session, simple_video, test_client):
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
def test_channel_conflicts(test_client, test_session, test_directory):
@pytest.mark.asyncio
async def test_channel_conflicts(test_async_client, test_session, test_directory):
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = dict(
@ -105,11 +107,11 @@ def test_channel_conflicts(test_client, test_session, test_directory):
url='https://example.com/channel1',
)
def _post_channel(channel):
return test_client.post('/api/videos/channels', content=json.dumps(channel))
async def _post_channel(channel):
return await test_async_client.post('/api/videos/channels', content=json.dumps(channel))
# Create it
request, response = _post_channel(new_channel)
request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.CREATED
# Name is an exact match
@ -119,7 +121,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
directory=channel_directory2,
name='Example Channel 1',
)
request, response = _post_channel(new_channel)
request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_NAME_CONFLICT',
'error': 'The channel name is already taken.',
@ -133,7 +135,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
directory=channel_directory,
name='name is fine',
)
request, response = _post_channel(new_channel)
request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_DIRECTORY_CONFLICT',
'error': 'The directory is already used by another channel.',
@ -150,7 +152,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
name='name is fine',
url='https://example.com/channel1',
)
request, response = _post_channel(new_channel)
request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_URL_CONFLICT',
'error': 'The URL is already used by another channel.',
@ -384,7 +386,8 @@ def test_search_videos_channel(test_client, test_session, video_factory):
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
@pytest.mark.asyncio
async def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
for i in range(50):
video_factory(channel_id=simple_channel.id)
@ -406,8 +409,9 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
]
last_ids = []
for offset, video_count in tests:
_, response = assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', offset=offset,
limit=20)
_, response = await assert_video_search(channel_id=simple_channel.id, order_by='published_datetime',
offset=offset,
limit=20)
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
current_ids = [i['id'] for i in response.json['file_groups']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
@ -416,7 +420,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
@pytest.mark.asyncio
async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory):
tag = tag_factory()
tag = await tag_factory()
# Create Download.
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
@ -465,7 +469,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
videos_directory = get_videos_directory()
channel = channel_factory(name='Channel Name', download_frequency=120)
channel_directory = channel.directory
tag = tag_factory('Tag Name')
tag = await tag_factory('Tag Name')
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
# Create recurring download which uses the Channel's directory.
download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
@ -532,7 +536,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
# Channel can be Tagged, but not move directories.
# Channel can be Tagged, without moving directories.
body = dict(tag_name=tag.name)
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
@ -545,3 +549,79 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
d1, d2 = test_session.query(Download).all()
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
@pytest.mark.asyncio
async def test_move_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory,
video_factory, test_channels_config, test_download_manager, test_downloader):
"""A Channel can be moved."""
channel = channel_factory(name='Channel Name')
vid1 = video_factory(channel_id=channel.id)
vid2 = video_factory(channel_id=channel.id)
test_session.commit()
assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
assert str(vid2.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
body = dict(
name=channel.name,
directory=str(test_directory / 'videos/New Directory'),
)
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = Channel.find_by_id(channel.id)
assert str(channel.directory) == str(test_directory / 'videos/New Directory')
assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/'))
assert str(vid2.video_path).startswith(str(test_directory / 'videos/New Directory/'))
@pytest.mark.asyncio
async def test_move_channel_after_terminal_move(test_async_client, test_session, test_directory, channel_factory,
tag_factory, video_factory, test_channels_config, test_download_manager,
test_downloader):
"""A Channel can be moved in the UI after being moved in the Terminal."""
channel = channel_factory(name='Channel Name')
vid1 = video_factory(channel_id=channel.id)
test_session.commit()
assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
# mv 'videos/Channel Name' 'videos/New Directory'
shutil.move(channel.directory, test_directory / 'videos/New Directory')
body = dict(
name=channel.name,
directory=str(test_directory / 'videos/New Directory'),
)
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = Channel.find_by_id(channel.id)
vid1 = test_session.query(Video).one()
assert str(channel.directory) == str(test_directory / 'videos/New Directory')
assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/'))
@pytest.mark.asyncio
async def test_search_tagged_channels(test_async_client, test_session, channel_factory, tag_factory,
video_factory):
"""Tagged Channels can be searched."""
tag = await tag_factory()
test_session.commit()
# Can search when empty.
body = dict(tag_names=[tag.name])
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 0, 'No Channels should be tagged'
channel1 = channel_factory(tag_name=tag.name)
channel2 = channel_factory()
test_session.commit()
body = dict(tag_names=[tag.name])
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 1, 'Only one Channel is tagged'
assert response.json['channels'][0]['id'] == channel1.id

View file

@ -1,6 +1,7 @@
from typing import List
import pytest
import yaml
from modules.videos import schema
from modules.videos.channel import lib
@ -124,3 +125,23 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact
await assert_search('foo', ['foo', 'foo bar', 'foobar'])
await assert_search('bar', ['foobar', 'foo bar', 'bar'], True)
@pytest.mark.asyncio
async def test_tag_channel_existing(test_async_client, test_session, test_directory, channel_factory, tag_factory,
video_factory, test_channels_config):
"""A Channel can already exist in a Tag's directory and still be tagged."""
channel_directory = test_directory / 'videos/one/Channel Name'
channel = channel_factory(name='Channel Name', download_frequency=120, directory=channel_directory)
video_factory(channel_id=channel.id)
test_session.commit()
assert str(channel.directory).startswith(f'{test_directory}/videos/one/')
tag = await tag_factory()
await lib.tag_channel(tag.name, channel_directory, channel.id, test_session)
# Channel's tag is saved to the config.
save_channels_config(test_session)
with test_channels_config.open() as fh:
config = yaml.load(fh, Loader=yaml.Loader)
assert config['channels'][0]['tag_name'] == tag.name

View file

@ -17,7 +17,7 @@ from sqlalchemy.orm import Session
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import FFPROBE_BIN
from wrolpi.common import logger, get_media_directory, get_wrolpi_config
from wrolpi.common import logger, get_media_directory
from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
@ -204,7 +204,7 @@ def is_valid_poster(poster_path: Path) -> bool:
async def ffprobe_json(video_path: Union[Path, str]) -> dict:
"""Extract video file metadata using ffprobe.
>>> ffprobe_json('video')
>>> ffprobe_json('video.mp4')
{
'chapters': [],
'format': {},
@ -270,39 +270,6 @@ def get_video_duration(video_path: Path) -> Optional[int]:
return duration
def check_for_video_corruption(video_path: Path) -> bool:
"""Uses ffprobe to check for specific ways a video file can be corrupt.
Also attempts to screenshot the video near the end of it's duration, this check if the video is corrupted somewhere
in the middle."""
video_path = Path(video_path) if not isinstance(video_path, Path) else video_path
if not video_path.is_file():
raise FileNotFoundError(f'{video_path} does not exist!')
if not FFPROBE_BIN:
raise SystemError('ffprobe is not installed!')
# Just read the video using FFMPEG, this prints out useful information about the video.
cmd = (FFPROBE_BIN, str(video_path))
try:
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
logger.warning(f'FFPROBE failed to check for corruption with stderr: {e.stderr.decode()}')
return True # video is corrupt.
messages = (
b'Invalid NAL unit size',
b'Error splitting the input into NAL units',
b'Invalid data found when processing input',
)
corrupt = False
for error in messages:
if error in proc.stderr:
logger.warning(f'Possible video corruption ({error.decode()}): {video_path}')
corrupt = True
return corrupt
async def update_view_counts_and_censored(channel_id: int):
"""Update view_count for all Videos in a channel using its info_json file. Also sets FileGroup.censored
if Video is no longer available on the Channel."""

View file

@ -40,14 +40,16 @@ def channel_factory(test_session, test_directory):
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
directory: pathlib.Path = None, tag_name: str = None):
name = name or str(uuid4())
tag = Tag.get_by_name(tag_name) if tag_name else None
tag = Tag.find_by_name(tag_name) if tag_name else None
channel = Channel(
name=name,
url=url or f'https://example.com/{name}',
source_id=source_id,
tag=tag,
tag_id=tag.id if tag else None,
)
channel.directory = directory or format_videos_destination(name, tag, channel.url)
tag_name = tag.name if tag else None
channel.directory = directory or format_videos_destination(name, tag_name, channel.url)
channel.directory.mkdir(exist_ok=True, parents=True)
test_session.add(channel)
test_session.flush([channel])
@ -242,12 +244,12 @@ def video_with_search_factory(test_session, test_directory, video_file_factory):
@pytest.fixture
def assert_video_search(test_client):
def assert_video_search(test_async_client):
"""A fixture which performs a video search request against the API.
If assert_* params are passed, the response is checked."""
def _search_videos(
async def _search_videos(
assert_total: int = None,
assert_ids: List[int] = None,
assert_paths: List[str] = None,
@ -272,7 +274,7 @@ def assert_video_search(test_client):
if channel_id is not None:
content['channel_id'] = channel_id
request, response = test_client.post('/api/videos/search', content=json.dumps(content))
request, response = await test_async_client.post('/api/videos/search', content=json.dumps(content))
assert response.json
assert response.status_code == HTTPStatus.OK

View file

@ -4,7 +4,7 @@ import os.path
import pathlib
import traceback
from abc import ABC
from typing import Tuple, List, Dict
from typing import Tuple, List, Dict, Optional
import yt_dlp.utils
from sqlalchemy.orm import Session
@ -64,6 +64,7 @@ def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict:
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
"""Get filename from YoutubeDL. Separated for testing."""
dir_name, file_name = os.path.split(ydl.prepare_filename(entry))
file_name = escape_file_name(file_name)
file_name = trim_file_name(file_name)
return f'{dir_name}/{file_name}'
@ -241,8 +242,8 @@ class VideoDownloader(Downloader, ABC):
# Video may have been downloaded previously, get its location for error reporting.
with get_db_session() as session:
videos = session.query(Video).join(FileGroup).filter_by(url=url).all()
location = videos[0].location if videos else None
video_ = Video.get_by_url(url, session)
location = video_.location if video_ else None
try:
download.info_json = download.info_json or extract_info(url)
@ -257,62 +258,7 @@ class VideoDownloader(Downloader, ABC):
if not download.info_json:
raise ValueError(f'Cannot download video with no info_json.')
found_channel = None
# Look for Channel using channel data first, fallback to uploader data.
channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
channel = None
if channel_name or channel_source_id or channel_url:
# Try to find the channel via info_json from yt-dlp.
try:
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
if channel:
found_channel = 'yt_dlp'
logger.debug(f'Found a channel with source_id {channel=}')
except UnknownChannel:
# Can't find a channel, use the no channel directory.
pass
settings = download.settings or dict()
destination = settings.get('destination')
if not channel and destination:
# Destination may find Channel if not already found.
try:
channel = get_channel(directory=destination, return_dict=False)
found_channel = 'download_settings_directory'
logger.debug(f'Found a channel that shares the destination directory {channel=}')
except UnknownChannel:
# Destination must not be a channel.
pass
local_channel_id = settings.get('channel_id')
channel_url = settings.get('channel_url')
if not channel and (local_channel_id or channel_url):
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
logger.info(f'Using download.settings to find channel')
try:
channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
logger.debug(f'Found channel with channel url {channel=}')
except UnknownChannel:
# We do not need a Channel if we have a destination directory.
if not destination:
raise
if channel:
found_channel = 'download_settings'
channel_id = channel.id if channel else None
channel_directory = channel.directory if channel else None
if found_channel == 'yt_dlp':
logger.debug(f'Found {channel} using yt_dlp')
elif found_channel == 'download_settings':
logger.info(f'Found {channel} using Download.settings')
elif found_channel == 'download_settings_directory':
logger.info(f'Found {channel} using Download.settings directory')
else:
logger.warning('Could not find channel')
channel, channel_directory, channel_id, destination, settings = await self._get_channel(download)
if destination:
# Download to the directory specified in the settings.
@ -480,6 +426,63 @@ class VideoDownloader(Downloader, ABC):
)
return result
@staticmethod
async def _get_channel(download: Download) \
-> Tuple[Optional[Channel], Optional[pathlib.Path], Optional[int], Optional[pathlib.Path], dict]:
found_channel = None
# Look for Channel using channel data first, fallback to uploader data.
channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
channel = None
if channel_name or channel_source_id or channel_url:
# Try to find the channel via info_json from yt-dlp.
try:
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
if channel:
found_channel = 'yt_dlp'
logger.debug(f'Found a channel with source_id {channel=}')
except UnknownChannel:
# Can't find a channel, use the no channel directory.
pass
settings = download.settings or dict()
destination = settings.get('destination')
if not channel and destination:
# Destination may find Channel if not already found.
try:
channel = get_channel(directory=destination, return_dict=False)
found_channel = 'download_settings_directory'
logger.debug(f'Found a channel that shares the destination directory {channel=}')
except UnknownChannel:
# Destination must not be a channel.
pass
channel_id = settings.get('channel_id')
channel_url = settings.get('channel_url')
if not channel and (channel_id or channel_url):
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
logger.info(f'Using download.settings to find channel')
try:
channel = get_channel(channel_id=channel_id, url=channel_url, return_dict=False)
logger.debug(f'Found channel with channel url {channel=}')
except UnknownChannel:
# We do not need a Channel if we have a destination directory.
if not destination:
raise
if channel:
found_channel = 'download_settings'
channel_id = channel.id if channel else None
channel_directory = channel.directory if channel else None
if found_channel == 'yt_dlp':
logger.debug(f'Found {channel} using yt_dlp')
elif found_channel == 'download_settings':
logger.info(f'Found {channel} using Download.settings')
elif found_channel == 'download_settings_directory':
logger.info(f'Found {channel} using Download.settings directory')
else:
logger.warning('Could not find channel')
return channel, channel_directory, channel_id, destination, settings
@staticmethod
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
"""Get the full path of a video file from its URL."""

View file

@ -3,6 +3,7 @@ import dataclasses
import html
import pathlib
import re
import traceback
from typing import Generator
from typing import Tuple, Optional
@ -15,7 +16,7 @@ from modules.videos.models import Video
from wrolpi import before_startup, dates, flags
from wrolpi.captions import extract_captions
from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \
escape_file_name, get_media_directory
escape_file_name, get_media_directory, background_task
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download
@ -379,6 +380,22 @@ def save_channels_config(session: Session = None):
channel_import_logger = logger.getChild('channel_import')
async def fetch_channel_source_id(channel_id: int):
# If we can download from a channel, we must have its source_id.
try:
with get_db_session() as session:
from modules.videos.channel.lib import get_channel
channel = Channel.find_by_id(channel_id)
channel.source_id = channel.source_id or get_channel_source_id(channel.url)
if not channel.source_id:
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
else:
session.commit()
save_channels_config(session)
except Exception as e:
logger.error(f'Failed to get Channel source id of id={channel_id}', exc_info=e)
@before_startup
@limit_concurrent(1)
def import_channels_config():
@ -387,13 +404,17 @@ def import_channels_config():
logger.warning('Skipping import_channels_config for this test')
return
from .channel.lib import get_channel
channel_import_logger.info('Importing channels config')
try:
config = get_channels_config()
channels = config.channels
channel_directories = [i['directory'] for i in channels]
if len(channel_directories) != len(set(channel_directories)):
logger.error('Refusing to import channels config because it contains duplicate directories!')
return
updated_channel_ids = set()
save_config = False
with get_db_session(commit=True) as session:
for data in channels:
if isinstance(data, str):
@ -405,45 +426,37 @@ def import_channels_config():
for option in (i for i in REQUIRED_OPTIONS if i not in data):
raise ConfigError(f'Channel "{directory}" is required to have "{option}"')
channel = session.query(Channel).filter(
or_(
Channel.directory == str(directory),
Channel.directory == str(data['directory']),
Channel.directory == str(directory.relative_to(get_media_directory())),
),
).one_or_none()
# Try to find Channel by directory because it is unique.
channel = Channel.get_by_path(directory, session)
if not channel:
# Channel not yet in the DB, add it.
channel = Channel(directory=directory)
session.add(channel)
session.flush([channel, ])
# TODO refresh the files in the channel.
logger.warning(f'Creating new Channel from config {channel}')
# Only name and directory are required
channel.name = data['name']
channel.directory = directory
# A URL should not be an empty string
data['url'] = data['url'] or None
try:
# Try to find Channel using other attributes before creating new Channel.
channel = get_channel(
session,
source_id=data.get('source_id'),
url=data.get('url'),
directory=str(directory),
return_dict=False,
)
except UnknownChannel:
# Channel not yet in the DB, add it.
channel = Channel(directory=directory)
session.add(channel)
channel.flush()
# TODO refresh the files in the channel.
logger.warning(f'Creating new Channel from config {channel}')
# Copy existing channel data, update all values from the config. This is necessary to clear out
# values not in the config.
full_data = channel.dict()
full_data.update(data)
channel.update(full_data)
updated_channel_ids.add(channel.id)
if not channel.source_id and channel.url and flags.have_internet.is_set():
# If we can download from a channel, we must have its source_id.
try:
channel.source_id = get_channel_source_id(channel.url)
save_config = True
if not channel.source_id:
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
else:
session.commit()
except Exception as e:
logger.error(f'Failed to get Channel source id of {channel}', exc_info=e)
logger.info(f'Fetching channel source id for {channel}')
background_task(fetch_channel_source_id(channel.id))
channel_import_logger.debug(f'Updated {repr(channel.name)}'
f' url={channel.url}'
@ -454,17 +467,13 @@ def import_channels_config():
with get_db_session(commit=True) as session:
# Delete any Channels that were deleted from the config.
for channel in session.query(Channel):
if str(channel.directory) not in channel_directories:
if channel.id not in updated_channel_ids:
logger.warning(f'Deleting {channel} because it is not in the config.')
channel.delete_with_videos()
# Create any missing Downloads. Associated Downloads with any necessary Channels.
link_channel_and_downloads()
if save_config:
# Information about the channel was fetched, store it.
save_channels_config()
except Exception as e:
channel_import_logger.warning('Failed to load channels config!', exc_info=e)
if PYTEST:
@ -544,7 +553,8 @@ async def get_statistics():
COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE ) AS "have_comments",
COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE
and fg.censored = false and fg.url is not null) AS "missing_comments",
COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments"
COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments",
COUNT(v.id) FILTER ( WHERE fg.censored = TRUE ) as "censored_videos"
FROM
video v
LEFT JOIN file_group fg on v.file_group_id = fg.id
@ -583,9 +593,11 @@ async def get_statistics():
curs.execute('''
SELECT
COUNT(id) AS "channels"
COUNT(c.id) AS "channels",
COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
FROM
channel
channel c
LEFT JOIN public.tag t on t.id = c.tag_id
''')
channel_stats = dict(curs.fetchone())
ret = dict(statistics=dict(

View file

@ -1,4 +1,5 @@
import json
import json
import pathlib
from typing import Optional, Dict, List
@ -7,11 +8,14 @@ from sqlalchemy.orm import relationship, Session, deferred
from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.errors import UnknownVideo, UnknownChannel
from wrolpi import flags
from wrolpi.captions import read_captions
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file, \
unique_by_predicate
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download, download_manager, save_downloads_config
from wrolpi.errors import InvalidDownload
from wrolpi.events import Events
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, move
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
@ -223,6 +227,7 @@ class Video(ModelHelper, Base):
# Can't validate if there is no video file.
logger.error(f'Unable to validate video {self.id} without primary file!')
logger.debug(f'Validating {self}')
from .lib import validate_video
try:
validate_video(self, self.channel.generate_posters if self.channel else False)
@ -237,12 +242,14 @@ class Video(ModelHelper, Base):
# self.file_group.d_text is handled in `validate_video`.
if (comments := self.get_comments()) and len(comments) >= 5:
logger.debug(f'{self} has comments')
self.have_comments = True
# If this Video is in a Channel's directory, then it is part of that Channel.
session: Session = Session.object_session(self)
if session and (channel := Channel.get_by_path(self.video_path.parent, session)):
self.channel = channel
logger.debug(f'{self} has Channel {channel}')
@staticmethod
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
@ -533,7 +540,7 @@ class Channel(ModelHelper, Base):
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'))
tag = relationship('Tag')
tag = relationship('Tag', primaryjoin='Channel.tag_id==Tag.id')
def __repr__(self):
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'
@ -565,12 +572,9 @@ class Channel(ModelHelper, Base):
channel_id = self.id
data = data.copy()
# Remove unused values.
data.pop('move_to_tag_directory', None)
# URL should not be empty string.
if 'url' in data:
data['url'] = data['url'] or None
url = data.pop('url')
self.url = url or None
session: Session = Session.object_session(self)
@ -664,23 +668,19 @@ class Channel(ModelHelper, Base):
with get_db_curs() as curs:
stmt = '''
SELECT
SUM(size),
MAX(size),
COUNT(video.id),
SUM(fg.length)
SUM(size) AS "size",
MAX(size) AS "largest_video",
COUNT(video.id) AS "video_count",
SUM(fg.length) AS "length",
-- Videos may use multiple tags.
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
FROM video
LEFT JOIN file_group fg on fg.id = video.file_group_id
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
WHERE channel_id = %(id)s
'''
curs.execute(stmt, dict(id=self.id))
size, largest_video, video_count, length = curs.fetchone()
statistics = dict(
video_count=video_count,
size=size,
largest_video=largest_video,
length=length,
)
return statistics
return dict(curs.fetchone())
async def refresh_files(self, send_events: bool = True):
"""Refresh all files within this Channel's directory. Mark this channel as refreshed."""
@ -772,39 +772,72 @@ class Channel(ModelHelper, Base):
channel = session.query(Channel).filter_by(url=url).one_or_none()
return channel
async def move(self, directory: pathlib.Path, session: Session):
def format_directory(self, tag_name: str) -> pathlib.Path:
from modules.videos.lib import format_videos_destination
return format_videos_destination(self.name, tag_name, self.url)
async def move_channel(self, directory: pathlib.Path, session: Session, send_events: bool = False):
"""Move the files of this Channel into a new directory."""
if not directory.is_dir():
raise FileNotFoundError(f'Destination directory does not exist: {directory}')
if self.directory and not next(self.directory.iterdir(), None):
raise FileExistsError(f'Refusing to move Channel when it has no files.')
old_directory = self.directory
self.directory = directory
session.flush([self, ])
# Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
# directory.
downloads = list(self.downloads)
downloads.extend(list(
session.query(Download).filter(Download.settings.contains({'destination': str(old_directory)}))
))
for download in downloads:
# Get new copy of `settings` to avoid bug where it wouldn't update.
settings = download.settings.copy() if download.settings else dict()
if settings.get('destination'):
download.settings = {**settings, 'destination': str(directory)}
def change_download_destinations(from_directory: pathlib.Path, to_directory: pathlib.Path):
downloads = list(self.downloads)
downloads.extend(Download.get_all_by_destination(from_directory))
downloads = unique_by_predicate(downloads, lambda i: i.id)
for download in downloads:
# Get new copy of `settings` to avoid bug where it wouldn't update.
settings = download.settings.copy() if download.settings else dict()
if settings.get('destination'):
logger.debug(f'Updating destination of {download}')
download.settings = {**settings, 'destination': str(to_directory)}
session.flush(downloads)
session.commit()
# Only one tag can be moved at a time.
with flags.refreshing:
# Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
# directory.
change_download_destinations(old_directory, directory)
# Save configs before move, this is because move imports configs.
from modules.videos.lib import save_channels_config
await save_downloads_config(session)
save_channels_config(session)
save_tags_config(session)
session.commit()
# Move the contents of the Channel directory into the destination directory.
await move(directory, *list(old_directory.iterdir()))
# Save configs before move, this is because move imports configs.
from modules.videos.lib import save_channels_config
await save_downloads_config(session)
save_channels_config(session)
save_tags_config(session)
if old_directory.exists() and not next(old_directory.iterdir(), None):
old_directory.rmdir()
# Move the contents of the Channel directory into the destination directory.
logger.info(f'Moving {self} from {repr(str(old_directory))}')
try:
if not old_directory.exists():
# Old directory does not exist, maintainer must have moved the Channel manually.
if PYTEST:
await refresh_files([old_directory, directory])
else:
background_task(refresh_files([old_directory, directory]))
if send_events:
Events.send_file_move_completed(f'Channel {repr(self.name)} was moved, but Tags were lost')
else:
# `move` also uses `flags.refreshing`
await move(directory, *list(old_directory.iterdir()))
if send_events:
Events.send_file_move_completed(f'Channel {repr(self.name)} was moved')
except Exception as e:
logger.error(f'Channel move failed! Reverting changes...', exc_info=e)
# Downloads must be moved back to the old directory.
change_download_destinations(directory, old_directory)
self.directory = old_directory
self.flush(session)
if send_events:
Events.send_file_move_failed(f'Moving Channel {self.name} has failed')
raise
finally:
session.commit()
if old_directory.exists() and not next(iter(old_directory.iterdir()), None):
# Old directory is empty, delete it.
old_directory.rmdir()

View file

@ -49,6 +49,7 @@ class ChannelTagRequest:
@dataclass
class ChannelTagInfoRequest:
channel_id: Optional[int] = None
tag_name: Optional[str] = None
@ -205,6 +206,7 @@ class VideoStatistics:
@dataclass
class ChannelStatistics:
channels: int
tagged_channels: int
@dataclass
@ -222,3 +224,8 @@ class CensoredVideoRequest:
@dataclass
class CensoredVideoResponse:
videos: List[VideoWithChannel]
@dataclass
class ChannelSearchRequest:
tag_names: List[str] = field(default_factory=list)

View file

@ -6,6 +6,7 @@ import pytest
from modules.videos.models import Video, Channel
from modules.videos.video import lib as video_lib
from wrolpi.common import get_wrolpi_config
from wrolpi.db import get_db_curs
from wrolpi.downloader import Download
from wrolpi.files.lib import refresh_files
@ -160,14 +161,15 @@ def test_api_download(test_session, test_client, test_directory):
assert not download.settings.get('tag_names')
def test_api_download_video_tags(test_session, test_client, tag_factory):
@pytest.mark.asyncio
async def test_api_download_video_tags(test_session, test_async_client, tag_factory):
"""A user can request Tags for a video being downloaded."""
tag1 = tag_factory()
tag2 = tag_factory()
tag1 = await tag_factory()
tag2 = await tag_factory()
content = dict(urls=['https://example.com/video1', ], downloader='video',
settings={'tag_names': [tag1.name, tag2.name]})
request, response = test_client.post('/api/download', content=json.dumps(content))
request, response = await test_async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
@ -176,7 +178,9 @@ def test_api_download_video_tags(test_session, test_client, tag_factory):
assert download.settings['tag_names'] == [tag1.name, tag2.name]
def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, assert_video_search):
@pytest.mark.asyncio
async def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory,
assert_video_search):
"""Test that videos can be searched and that their order is by their textsearch rank."""
# These captions have repeated letters, so they will be higher in the ranking.
videos = [
@ -193,37 +197,38 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
# Repeated runs should return the same result
for _ in range(2):
# Only videos with a b are returned, ordered by the amount of b's
assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
# Only two captions have e
assert_video_search(search_str='e', assert_ids=[4, 1])
await assert_video_search(search_str='e', assert_ids=[4, 1])
# Only 2 contains 2
assert_video_search(search_str='2', assert_ids=[2, ])
await assert_video_search(search_str='2', assert_ids=[2, ])
# Only two captions have d
assert_video_search(search_str='d', assert_ids=[1, 2])
await assert_video_search(search_str='d', assert_ids=[1, 2])
# 5 can be gotten by it's title
assert_video_search(search_str='5', assert_ids=[5, ])
await assert_video_search(search_str='5', assert_ids=[5, ])
# only video 1 has e and d
assert_video_search(search_str='e d', assert_ids=[1, ])
await assert_video_search(search_str='e d', assert_ids=[1, ])
# video 1 and 4 have b and e, but 1 has more
assert_video_search(search_str='b e', assert_ids=[1, 4])
await assert_video_search(search_str='b e', assert_ids=[1, 4])
# Check totals are correct even with a limit
assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
@pytest.mark.asyncio
async def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
"""Search the Video table. This does not need to use a join with the File table."""
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2',
channel_id=simple_channel.id)
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
vid1.add_tag(tag1.id)
@ -237,22 +242,68 @@ def test_search_videos(test_session, video_factory, assert_video_search, simple_
assert test_session.query(Video).count() == 3
assert test_session.query(FileGroup).count() == 3
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2,
offset=1)
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
channel_id=simple_channel.id)
assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
await assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2,
offset=1)
await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
channel_id=simple_channel.id)
await assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
await assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
# Only vid2 has both tags.
assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
await assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
# No results, no total is returned from the DB.
assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
await assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
# Check all order_by.
for order_by in video_lib.VIDEO_ORDERS:
assert_video_search(order_by=order_by)
await assert_video_search(order_by=order_by)
@pytest.mark.asyncio
async def test_delete_video(test_async_client, test_session, video_factory, test_download_manager):
"""When a Video record is deleted, the FileGroup should be deleted. The URL is added to the
global skip list so the video is not downloaded again."""
video = video_factory(with_video_file=True, with_poster_ext='png')
video.file_group.url = 'https://example.com/video/1'
files = video.file_group.my_paths()
test_session.commit()
request, response = await test_async_client.delete(f'/api/videos/video/{video.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
assert test_session.query(FileGroup).count() == 0, 'Video FileGroup was not deleted.'
assert not any(i.exists() for i in files), 'All files should have been deleted.'
assert test_download_manager.is_skipped('https://example.com/video/1'), 'Video should not be downloaded again.'
@pytest.mark.asyncio
async def test_format_videos_description(test_async_client, test_session, test_directory, channel_factory, tag_factory):
channel = channel_factory(name='Channel Name')
tag = await tag_factory()
test_session.commit()
body = dict()
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos'
body = dict(channel_id=channel.id)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/Channel Name'
body = dict(channel_id=channel.id, tag_name=tag.name)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/one/Channel Name'
get_wrolpi_config().videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
body = dict(channel_id=channel.id, tag_name=tag.name)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'

View file

@ -135,10 +135,12 @@ def test_generate_video_poster(video_file):
@pytest.mark.asyncio
async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config):
async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config,
tag_factory):
"""Importing the Channels' config should create any missing download records"""
channel1 = channel_factory(source_id='foo', url='https://example.com/channel1')
channel2 = channel_factory(source_id='bar', url='https://example.com/channel2')
tag = await tag_factory()
assert len(test_session.query(Channel).all()) == 2
test_session.commit()
@ -228,8 +230,19 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
assert {i.url for i in channel1.downloads} == {'https://example.com/channel1'}
assert {i.url for i in channel2.downloads} == {'https://example.com/channel2', 'https://example.org'}
# A Channel with a `tag_name` is tagged.
update_channel_config(get_channels_config(),
channel1.source_id,
{'tag_name': tag.name})
import_channels_config()
channel1, channel2 = test_session.query(Channel).order_by(Channel.url).all()
assert channel1.tag == tag and channel1.tag_id == tag.id
assert channel2.tag is None and channel2.tag_id is None
def test_import_channel_delete_missing_channels(test_session, channel_factory, test_channels_config):
@pytest.mark.asyncio
async def test_import_channel_delete_missing_channels(
test_async_client, test_session, channel_factory, test_channels_config):
"""The Channel import function deletes any Channels that are not in the config."""
# Create a DB and config with two Channels.
channel1 = channel_factory(source_id='foo')
@ -268,32 +281,6 @@ def test_import_channel_delete_missing_channels(test_session, channel_factory, t
assert str(channel2.directory) not in test_channels_config.read_text()
def test_check_for_video_corruption(video_file, test_directory):
# The test video is not corrupt.
assert common.check_for_video_corruption(video_file) is False
# An empty file is corrupt.
empty_file = test_directory / 'empty file.mp4'
empty_file.touch()
assert common.check_for_video_corruption(empty_file) is True
# A video file must be complete.
truncated_video = test_directory / 'truncated_video.mp4'
with truncated_video.open('wb') as fh:
fh.write(video_file.read_bytes()[:10000])
assert common.check_for_video_corruption(truncated_video) is True
# Check for specific ffprobe errors.
with mock.patch('modules.videos.common.subprocess') as mock_subprocess:
# `video_file` is ignored for these calls.
mock_subprocess.run().stderr = b'Something\nInvalid NAL unit size'
assert common.check_for_video_corruption(video_file) is True
mock_subprocess.run().stderr = b'Something\nError splitting the input into NAL units'
assert common.check_for_video_corruption(video_file) is True
mock_subprocess.run().stderr = b'Some stderr is fine'
assert common.check_for_video_corruption(video_file) is False
@pytest.mark.asyncio
async def test_ffprobe_json(test_async_client, video_file, corrupted_video_file):
content = await common.ffprobe_json(video_file)

View file

@ -107,7 +107,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
mock_video_extract_info, mock_video_process_runner, tag_factory):
"""A Video is tagged when Download record requires it."""
video_path = video_file_factory()
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
mock_video_extract_info.return_value = copy(example_video_json)

View file

@ -34,7 +34,7 @@ async def test_channel_move(test_async_client, test_session, test_directory, cha
foo.mkdir(parents=True)
# Move the Channel.
await channel.move(foo, test_session)
await channel.move_channel(foo, test_session)
assert channel.directory == foo
assert channel.directory != (test_directory / 'New Channel Directory')

View file

@ -29,27 +29,11 @@ def test_delete_video_no_channel(test_session, simple_video):
assert test_session.query(Video).count() == 0
def test_delete_video(test_session, video_factory):
"""When a Video record is deleted, all referenced file records should be deleted."""
video = video_factory(with_video_file=True, with_poster_ext='png')
test_session.commit()
assert video.video_path.is_file(), 'Video file was not created.'
assert video.poster_path.is_file(), 'Video poster was not created.'
assert test_session.query(Video).count() == 1, 'Video was not created.'
assert test_session.query(FileGroup).count() == 1, 'Video file and poster file were not created.'
video.delete()
test_session.commit()
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.'
@pytest.mark.asyncio
async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory):
"""You cannot delete a video with a tag."""
video = video_factory(with_video_file=True, with_poster_ext='png')
tag = tag_factory()
tag = await tag_factory()
video.add_tag(tag.name)
test_session.commit()
@ -63,7 +47,7 @@ async def test_delete_video_with_tag(test_async_client, test_session, video_fact
@pytest.mark.asyncio
async def test_video_channel_refresh(test_session, test_directory, channel_factory, video_factory):
async def test_video_channel_refresh(test_async_client, test_session, test_directory, channel_factory, video_factory):
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
# Create a video file in this channel's directory.
channel = channel_factory()
@ -101,7 +85,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
entry = channel.info_json['entries'][0]
tag = tag_factory()
tag = await tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
@ -150,7 +134,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
@pytest.mark.asyncio
async def test_delete_renamed_video(test_session, channel_factory, video_factory, tag_factory):
async def test_delete_renamed_video(test_async_client, test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'
@ -158,7 +142,7 @@ async def test_delete_renamed_video(test_session, channel_factory, video_factory
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]}
entry = channel.info_json['entries'][0]
tag = tag_factory()
tag = await tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',

View file

@ -164,8 +164,11 @@ def delete_videos(*video_ids: int, session: Session = None):
raise UnknownVideo('Could not find videos to delete')
logger.warning(f'Deleting {len(videos)} videos')
# Get all URLs, skip them once (so config isn't saved multiple times).
urls = list(filter(None, [i.file_group.url for i in videos]))
download_manager.add_to_skip_list(*urls)
for video in videos:
video.delete(add_to_skip_list=True)
video.delete()
session.commit()

View file

@ -9,7 +9,7 @@ from modules.videos.models import Video
from wrolpi.dates import now
def test_get_video_prev_next(test_session, channel_factory, video_factory):
def test_get_video_prev_next(test_async_client, test_session, channel_factory, video_factory):
"""
Test that the previous and next videos will be retrieved when fetching a video.
"""
@ -69,7 +69,7 @@ def test_get_video_prev_next(test_session, channel_factory, video_factory):
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
def test_get_video_prev_next_no_upload_date(test_session, video_factory, channel_factory):
def test_get_video_prev_next_no_upload_date(test_async_client, test_session, video_factory, channel_factory):
"""
Previous and Next Videos should be those next to a Video file when videos do not have upload dates.
"""
@ -153,7 +153,7 @@ async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_m
test_download_manager, tag_factory):
"""Many methods are blocked when WROL Mode is enabled."""
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
tag = tag_factory()
tag = await tag_factory()
await wrol_mode_fixture(True)

View file

@ -100,14 +100,14 @@ def zim_factory(test_session, zim_path_factory, test_directory):
@pytest.fixture
def assert_zim_search(test_client, test_directory):
def assert_zim_search(test_async_client, test_directory):
from http import HTTPStatus
def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
expected_status_code: int = HTTPStatus.OK):
async def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
expected_status_code: int = HTTPStatus.OK):
content = dict(search_str=search_str, tag_names=tag_names)
assert isinstance(zim_id, int), 'This fixture only supports Zim ids of type integer'
request, response = test_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
request, response = await test_async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
assert response.status_code == expected_status_code
result = response.json['zim']

View file

@ -16,7 +16,7 @@ from modules.zim.kiwix import KIWIX_CATALOG
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
from wrolpi import flags
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
register_refresh_cleanup, background_task, get_wrolpi_config
register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate
from wrolpi.db import get_db_session, optional_session, get_db_curs
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import refresh_files, split_file_name_words
@ -124,14 +124,11 @@ def get_entries_tags(paths: List[str], zim_id: int):
def get_unique_paths(*paths: str) -> List[str]:
"""Return a new list which contains unique Zim paths."""
new_paths = list()
simple_paths = list()
for path in paths:
simple_path = path.replace('-', '').replace('_', '').replace('/', '').lower()
if simple_path not in simple_paths:
new_paths.append(path)
simple_paths.append(simple_path)
return new_paths
unique_paths = unique_by_predicate(
paths,
lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower()
)
return list(unique_paths)
@optional_session

View file

@ -8,16 +8,17 @@ from modules.zim.models import Zim
from wrolpi.downloader import Download
def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim_search):
@pytest.mark.asyncio
async def test_zim_search(test_async_client, test_session, test_zim, tag_factory, assert_zim_search):
"""Entries can be tagged and retrieved using their containing Zim."""
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
test_session.commit()
# Test searching with only `search_str`.
assert_zim_search('one', 1, {
await assert_zim_search('one', 1, {
'path': test_zim.path,
'search': [
# Entry "One" contains a 'one' in the title, and the text.
@ -27,13 +28,13 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
],
'estimate': 2,
})
assert_zim_search('two', 1, {
await assert_zim_search('two', 1, {
'path': test_zim.path,
'search': [{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
{'path': 'home', 'headline': 'testing.\nOne\n<b>Two</b>', 'rank': 0.06079271}],
'estimate': 2,
})
assert_zim_search('item', 1, {
await assert_zim_search('item', 1, {
'path': test_zim.path,
'search': [{'path': 'one', 'headline': 'first <b>item</b>', 'rank': 0.06079271},
{'path': 'two', 'headline': 'second <b>item</b>', 'rank': 0.06079271}],
@ -41,35 +42,35 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
})
# Test searching with `search_str` and `tag_names`.
assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={
await assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={
'search': [
# Only entry "One" is tagged with Tag 2.
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
],
'estimate': 1,
})
assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={
await assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={
'search': [
# Only "One" is tagged with Tag 1 and contains "one".
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
],
'estimate': 1,
})
assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={
await assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={
'search': [
# Only "Two" was tagged with Tag 1 and contains "two".
{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
],
'estimate': 1,
})
assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={
await assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={
# Two was not tagged with Tag 1.
'search': [],
'estimate': 0,
})
# Test search with only `tag_names`.
assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={
await assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={
'search': [
# Return all entries that are tagged with the Tags.
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.0, 'tag_names': ['one', 'two']},
@ -78,9 +79,10 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
})
def test_entries_by_tag(test_client, test_session, test_zim, tag_factory):
@pytest.mark.asyncio
async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_factory):
"""Entries can be tagged and retrieved using their containing Zim."""
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
@ -96,36 +98,38 @@ def test_entries_by_tag(test_client, test_session, test_zim, tag_factory):
assert [(i.path, i.title) for i in result] == [('one', 'One'), ]
def test_zim_crud(test_client, test_session, test_directory, test_zim):
request, response = test_client.get('/api/zim')
@pytest.mark.asyncio
async def test_zim_crud(test_async_client, test_session, test_directory, test_zim):
request, response = await test_async_client.get('/api/zim')
assert response.status_code == HTTPStatus.OK
assert len(response.json['zims']) == 1
assert response.json['zims'][0]['path'] == test_zim.path.name
request, response = test_client.delete('/api/zim/1')
request, response = await test_async_client.delete('/api/zim/1')
assert response.status_code == HTTPStatus.NO_CONTENT
assert not test_zim.path.exists()
assert test_session.query(Zim).count() == 0
def test_zim_tag_and_untag(test_client, test_session, test_zim, tag_factory):
@pytest.mark.asyncio
async def test_zim_tag_and_untag(test_async_client, test_session, test_zim, tag_factory):
"""Zim entries can be Tagged and Untagged."""
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
test_session.commit()
# Entries can be tagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = test_client.post('/api/zim/tag', content=json.dumps(content))
request, response = await test_async_client.post('/api/zim/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
# Entries can be untagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Cannot untag twice.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NOT_FOUND

View file

@ -51,14 +51,14 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
@pytest.mark.asyncio
async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory):
tag1 = tag_factory()
tag1 = await tag_factory()
with pytest.raises(UnknownZimEntry):
test_zim.tag_entry(tag1.name, 'path does not exist')
@pytest.mark.asyncio
async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory):
tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
@ -70,7 +70,8 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t
@pytest.mark.asyncio
async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory):
tze = zim_factory('zim1').tag_entry(tag_factory().name, 'one')
tag = await tag_factory()
tze = zim_factory('zim1').tag_entry(tag.name, 'one')
# Test TagZimEntry.__repr__
assert repr(tze) == "<TagZimEntry tag='one' zim=zim1 zim_entry=one>"
@ -78,7 +79,7 @@ async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory,
@pytest.mark.asyncio
async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory):
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
# Tag Zim1 with both tags, but Zim2 shares one of the tags and is returned when no Zim is passed.
zim1.tag_entry(tag1.name, 'one')
@ -124,13 +125,14 @@ def test_get_unique_paths():
'A/Thomas_Harvey_Johnston']
def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
@pytest.mark.asyncio
async def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
config = tags.get_tags_config()
assert not config.tag_zims
# Tag three Zim entries.
tag1, tag2 = tag_factory('Tag1'), tag_factory('Tag2')
tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2')
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
@ -243,12 +245,13 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te
assert not flags.outdated_zims.is_set()
def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
@pytest.mark.asyncio
async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
# Tag outdated Zim entries.
zim1.tag_entry(tag1.name, 'home')
zim1.tag_entry(tag2.name, 'one')

View file

@ -14,6 +14,7 @@ import socket
import string
import sys
import tempfile
import traceback
from asyncio import Task
from copy import deepcopy
from dataclasses import dataclass
@ -160,6 +161,7 @@ __all__ = [
'ConfigFile',
'DownloadFileInfo',
'DownloadFileInfoLink',
'LOGGING_CONFIG',
'ModelHelper',
'WROLPI_CONFIG',
'aiohttp_get',
@ -180,7 +182,6 @@ __all__ = [
'compile_tsvector',
'cum_timer',
'date_range',
'LOGGING_CONFIG',
'disable_wrol_mode',
'download_file',
'enable_wrol_mode',
@ -227,6 +228,7 @@ __all__ = [
'truncate_generator_bytes',
'truncate_object_bytes',
'tsvector',
'unique_by_predicate',
'url_strip_host',
'walk',
'wrol_mode_check',
@ -252,10 +254,6 @@ class ModelHelper:
def get_by_path(path, session) -> Base:
raise NotImplementedError('This model has not defined this method.')
@staticmethod
def find_by_paths(paths, session) -> List:
raise NotImplementedError('This model has not defined this method.')
@staticmethod
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
"""Attempts to get a model instance by its id. Returns None if no instance can be found."""
@ -274,9 +272,10 @@ class ModelHelper:
def do_model(file_group, session: Session):
raise NotImplementedError('This model has not defined this method.')
def flush(self):
def flush(self, session: Session = None):
"""A convenience method which flushes this record using its DB Session."""
if session := Session.object_session(self):
session = session or Session.object_session(self)
if session:
session.flush([self])
else:
raise RuntimeError(f'{self} is not in a session!')
@ -746,6 +745,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
synchronous_after = after
async def after(*a, **kw):
if logger.isEnabledFor(logging.DEBUG):
traceback.print_stack()
logger.debug(f'run_after synchronous_after called for {after}')
return synchronous_after(*a, **kw)
def wrapper(func: callable):
@ -756,6 +758,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
@wraps(func)
async def wrapped(*a, **kw):
results = await func(*a, **kw)
if logger.isEnabledFor(logging.DEBUG):
traceback.print_stack()
logger.debug(f'run_after async called for {func}')
coro = after(*args, **kwargs)
asyncio.ensure_future(coro)
return results
@ -763,6 +768,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
@wraps(func)
def wrapped(*a, **kw):
results = func(*a, **kw)
if logger.isEnabledFor(logging.DEBUG):
traceback.print_stack()
logger.debug(f'run_after sync called for {func}')
coro = after(*args, **kwargs)
asyncio.ensure_future(coro)
return results
@ -1741,6 +1749,35 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
return headlines
async def search_other_estimates(tag_names: List[str]) -> dict:
"""Estimate other things that are Tagged."""
from wrolpi.db import get_db_curs
if not tag_names:
return dict(
channel_count=0,
)
with get_db_curs() as curs:
stmt = '''
SELECT
COUNT(c.id)
FROM channel c
LEFT OUTER JOIN public.tag t on t.id = c.tag_id
WHERE t.name = %(tag_name)s
'''
# TODO handle multiple tags
params = dict(tag_name=tag_names[0])
curs.execute(stmt, params)
channel_count = curs.fetchone()[0]
others = dict(
channel_count=channel_count,
)
return others
def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False):
"""Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful.
The original file will be restored if any of these steps fail."""
@ -1891,3 +1928,29 @@ def is_valid_hex_color(hex_color: str) -> bool:
def is_hardlinked(path: pathlib.Path) -> bool:
"""Returns True only if `path` has more than one link."""
return path.stat().st_nlink > 1
def _default_predicate(i):
return i
def unique_by_predicate(
iterable: list | tuple | set,
predicate: callable = None,
) -> list | tuple | set:
"""Finds only unique items in the provided iterable based on the predicate function. Order is preserved.
`predicate` defaults to the items themselves.
>>> unique_by_predicate([['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach']], lambda i: len(i))
# [ 'apple', 'banana', 'pear' ]
>>> unique_by_predicate([Path('foo.txt'), Path('foo.mp4'), Path('bar.txt')], lambda i: i.stem)
# [ Path('foo.txt'), Path('bar.txt') ]
>>> unique_by_predicate([1, 1, 2, 3, 3, 3, 3, 4], None)
# [ 1, 2, 3, 4 ]
"""
predicate = predicate or _default_predicate # Return an iterable based off the object itself by default.
seen = set()
unique_items = [i for i in iterable if not ((key := predicate(i)) in seen or seen.add(key))]
# Return the same type as the input iterable
return iterable.__class__(unique_items) # For other iterable types

View file

@ -16,7 +16,7 @@ import zipfile
from abc import ABC
from datetime import datetime
from itertools import zip_longest
from typing import List, Callable, Dict, Sequence, Union, Coroutine
from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional
from typing import Tuple, Set
from unittest import mock
from unittest.mock import MagicMock, AsyncMock
@ -175,7 +175,7 @@ async def test_async_client(test_directory) -> SanicASGITestClient:
@pytest.fixture
def test_download_manager_config(test_async_client, test_directory):
def test_download_manager_config(test_async_client, test_directory) -> pathlib.Path:
with downloads_manager_config_context():
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/download_manager.yaml'
@ -187,7 +187,7 @@ async def test_download_manager(
test_async_client,
test_session, # session is required because downloads can start without the test DB in place.
test_download_manager_config,
):
) -> DownloadManager:
# Needed to use signals in test app?
api_app.signalize()
@ -200,7 +200,7 @@ async def test_download_manager(
@pytest.fixture
def fake_now():
def fake_now() -> Callable:
try:
set_test_now(datetime(2000, 1, 1))
yield set_test_now
@ -210,17 +210,17 @@ def fake_now():
@pytest.fixture
def successful_download():
def successful_download() -> DownloadResult:
return DownloadResult(success=True)
@pytest.fixture
def failed_download():
return DownloadResult(error='pytest.fixture failed_download error', success=False)
def failed_download() -> DownloadResult:
return DownloadResult(error='wrolpi.conftest failed_download error', success=False)
@pytest.fixture
def assert_download_urls(test_session):
def assert_download_urls(test_session) -> Callable[[Set[str]], None]:
def asserter(expected_urls: Set[str]):
downloads = test_session.query(Download).all()
urls = {i.url for i in downloads}
@ -231,7 +231,7 @@ def assert_download_urls(test_session):
@pytest.fixture
def assert_downloads(test_session):
def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
from wrolpi.test.common import assert_dict_contains
def asserter(expected: List[Dict]):
@ -245,7 +245,7 @@ def assert_downloads(test_session):
@pytest.fixture
def test_downloader(test_download_manager):
def test_downloader(test_download_manager) -> Downloader:
class TestDownloader(Downloader, ABC):
"""A testing Downloader"""
name = 'test_downloader'
@ -314,7 +314,7 @@ def corrupted_video_file(test_directory) -> pathlib.Path:
@pytest.fixture
def video_file_factory(test_directory):
def video_file_factory(test_directory) -> Callable[[pathlib.Path], pathlib.Path]:
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
def _(path: pathlib.Path = None) -> pathlib.Path:
@ -327,7 +327,7 @@ def video_file_factory(test_directory):
@pytest.fixture
def video_bytes():
def video_bytes() -> bytes:
return (PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()
@ -467,7 +467,7 @@ def wrol_mode_fixture(test_config, test_download_manager) -> Callable[[bool], Co
@pytest.fixture
def mock_create_subprocess_shell():
def mock_create_subprocess_shell() -> Callable:
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
async def communicate(*a, **kw):
if communicate_side_effect:
@ -507,14 +507,14 @@ def flags_lock():
@pytest.fixture()
def tag_factory(test_session):
def tag_factory(test_session) -> Callable[[Optional[str]], Awaitable[Tag]]:
names = ['one', 'two', 'three', 'four', 'five', 'six']
count = 1
def factory(name: str = None) -> Tag:
async def factory(name: str = None) -> Tag:
if not name:
name = names.pop(0)
tag = upsert_tag(name, f'#{str(count) * 6}')
tag = await upsert_tag(name, f'#{str(count) * 6}', session=test_session)
return tag
return factory

View file

@ -52,6 +52,7 @@ def attach_shared_contexts(app: Sanic):
# Locks
app.shared_ctx.config_save_lock = multiprocessing.Lock()
app.shared_ctx.events_lock = multiprocessing.Lock()
reset_shared_contexts(app)

View file

@ -253,6 +253,15 @@ class Download(ModelHelper, Base): # noqa
return download
raise UnknownDownload(f'Cannot find Download with URL: {url}')
@classmethod
@optional_session
def get_all_by_destination(cls, destination: str | pathlib.Path, session: Session = None) -> List['Download']:
destination = str(destination)
downloads = session.query(Download) \
.filter(Download.settings.contains({'destination': destination})) \
.all()
return downloads
class Downloader:
name: str = None
@ -1330,7 +1339,7 @@ class RSSDownloader(Downloader, ABC):
# Only download new URLs.
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]]
logger.info(f'Successfully got {len(urls)} urls from rss {download.url}')
logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}')
return DownloadResult(success=True, downloads=urls)

View file

@ -10,7 +10,6 @@ from wrolpi.dates import now
logger = logger.getChild(__name__)
HISTORY_SIZE = 100
EVENTS_LOCK = multiprocessing.Lock()
class Events:
@ -79,6 +78,14 @@ class Events:
def send_shutdown_failed(message: str = None):
send_event('shutdown_failed', message, subject='shutdown')
@staticmethod
def send_file_move_completed(message: str = None):
send_event('file_move_completed', message, subject='refresh')
@staticmethod
def send_file_move_failed(message: str = None):
send_event('file_move_failed', message, subject='refresh')
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
log = f'{event=}'
@ -93,7 +100,7 @@ def log_event(event: str, message: str = None, action: str = None, subject: str
def send_event(event: str, message: str = None, action: str = None, subject: str = None, url: str = None):
from wrolpi.api_utils import api_app
EVENTS_LOCK.acquire()
api_app.shared_ctx.events_lock.acquire()
try:
# All events will be in time order, they should never be at the exact same time.
dt = now()
@ -112,7 +119,7 @@ def send_event(event: str, message: str = None, action: str = None, subject: str
while len(api_app.shared_ctx.events_history) > HISTORY_SIZE:
api_app.shared_ctx.events_history.pop(0)
finally:
EVENTS_LOCK.release()
api_app.shared_ctx.events_lock.release()
log_event(event, message, action, subject)

View file

@ -12,6 +12,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
from . import lib, schema
from ..api_utils import json_response, api_app
from ..events import Events
from ..schema import JSONErrorResponse
from ..tags import Tag
from ..vars import PYTEST
@ -224,7 +225,9 @@ async def post_move(_: Request, body: schema.Move):
try:
await lib.move(destination, *sources)
Events.send_file_move_completed(f'Moving files to {body.destination} succeeded')
except Exception as e:
Events.send_file_move_failed(f'Moving files to {body.destination} failed!')
raise FileConflict(f'Failed to move {sources} to {destination}') from e
return response.empty(HTTPStatus.NO_CONTENT)
@ -245,7 +248,7 @@ async def post_rename(_: Request, body: schema.Rename):
raise FileConflict(f'File already exists: {new_path}')
try:
await lib.rename(path, body.new_name)
await lib.rename(path, body.new_name, send_events=True)
except Exception as e:
logger.error(f'Failed to rename {path} to {new_path}', exc_info=e)
raise FileConflict(f'Failed to rename {path} to {new_path}') from e
@ -297,14 +300,15 @@ async def post_upload(request: Request):
Will not overwrite an existing file, unless a previous upload did not complete."""
try:
destination = request.form['destination'][0]
destination_str = request.form['destination'][0]
except Exception as e:
logger.error(f'Failed to get upload destination', exc_info=e)
raise UnknownDirectory('Must provide destination') from e
destination = get_media_directory() / destination
if not destination.is_dir():
raise UnknownDirectory(f'Destination must be a relative directory that is already in the media directory: {destination}')
destination = get_media_directory() / destination_str
if destination_str.startswith('/') or destination_str.startswith('.') or not destination.is_dir():
msg = f'Destination must be a relative directory that is already in the media directory: {destination}'
raise UnknownDirectory(msg)
try:
filename = str(request.form['filename'][0])
@ -385,7 +389,11 @@ async def post_upload(request: Request):
logger.info(f'Got final chunk of uploaded file: {output_str}')
del api_app.shared_ctx.uploaded_files[output_str]
# Upsert this new file (and any related files) into the DB.
background_task(lib.upsert_file(output, tag_names=tag_names))
coro = lib.upsert_file(output, tag_names=tag_names)
if PYTEST:
await coro
else:
background_task(coro)
return response.empty(HTTPStatus.CREATED)
# Request the next chunk.

View file

@ -9,13 +9,15 @@ import pathlib
import re
import shutil
import subprocess
from collections import OrderedDict
from itertools import zip_longest
from pathlib import Path
from typing import List, Tuple, Union, Dict, Generator, Iterable
from typing import List, Tuple, Union, Dict, Generator, Iterable, Set
import cachetools.func
import psycopg2
from sqlalchemy import asc
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from wrolpi import flags
@ -24,7 +26,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
partition, cancelable_wrapper, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
get_wrolpi_config, \
timer
timer, chunks, unique_by_predicate
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
@ -32,7 +34,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
from wrolpi.events import Events
from wrolpi.files.models import FileGroup, Directory
from wrolpi.lang import ISO_639_CODES
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config
from wrolpi.vars import PYTEST
try:
@ -48,7 +50,7 @@ logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file',
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
@ -311,6 +313,12 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = Fals
return path.name, ''
def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \
-> List[pathlib.Path]:
"""Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem."""
return unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0])
refresh_logger = logger.getChild('refresh')
@ -578,6 +586,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
refreshing_all_files = False
with flags.refreshing, timer('refresh_files', 'info'):
api_app.shared_ctx.refresh['counted_files'] = 0
if not paths:
refresh_logger.warning('Refreshing all files')
refreshing_all_files = True
@ -603,8 +612,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
continue
directories.extend(dirs)
found_directories |= set(dirs)
api_app.shared_ctx.refresh['counted_files'] = api_app.shared_ctx.refresh.get('counted_files',
0) + len(files)
add_files_to_refresh_count(len(files))
# Sleep to catch cancel.
await asyncio.sleep(0)
refresh_logger.info(f'Counted {api_app.shared_ctx.refresh["counted_files"]} files')
@ -633,7 +641,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
with get_db_session() as session:
parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))}
parent_directories |= set(filter(lambda i: i.is_dir(), paths))
await upsert_directories(parent_directories, found_directories)
upsert_directories(parent_directories, found_directories)
if send_events:
Events.send_global_after_refresh_completed()
@ -684,7 +692,7 @@ async def apply_indexers():
break
async def upsert_directories(parent_directories, directories):
def upsert_directories(parent_directories, directories):
"""
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
`directories`.
@ -1168,7 +1176,17 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
return 0
def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
MOVE_CHUNK_SIZE = 100
def add_files_to_refresh_count(count: int):
from wrolpi.api_utils import api_app
count = api_app.shared_ctx.refresh.get('counted_files', 0) + count
api_app.shared_ctx.refresh['counted_files'] = count
async def _move(destination: pathlib.Path, *sources: pathlib.Path, session: Session) \
-> OrderedDict[pathlib.Path, pathlib.Path]:
media_directory = get_media_directory()
for source in sources:
if not str(source).startswith(str(media_directory)):
@ -1177,70 +1195,155 @@ def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathl
raise FileNotFoundError(f'{destination} is not within the media directory')
destination_existed = destination.is_dir()
destination.mkdir(parents=True, exist_ok=True)
# [ (pathlib.Path('source'), pathlib.Path('destination')), ... ]
plan = list()
unique_paths = set()
# Get all files that need to be moved. Check for destination conflicts.
for source in sources:
if source.is_dir():
# Recursively move the directory's files.
for path in walk(source):
# Preserve the directory structure that contains this file.
new_file = destination / source.name / path.relative_to(source)
if new_file.exists():
raise FileConflict(f'Cannot move {path} to {new_file} because it already exists')
if path not in unique_paths:
plan.append((path, new_file))
unique_paths.add(path)
elif source.is_file():
# Move all files associated with the source file.
for file in glob_shared_stem(source):
new_file = destination / file.name
if new_file.exists():
raise FileConflict(f'Cannot move {file} to {new_file} because it already exists')
if file not in unique_paths:
plan.append((file, new_file))
unique_paths.add(file)
# The files that will be moved. [ (old_file, new_file), ... ]
plan: Dict[pathlib.Path, pathlib.Path] = dict()
logger.info(f'Executing move plan with {len(plan)} steps.')
def add_file_group_to_plan(file_group_: FileGroup):
primary_path_ = file_group_.primary_path
if primary_path_ not in plan:
new_primary_path_ = destination / primary_path_.name
if new_primary_path_.exists():
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
plan[primary_path_] = new_primary_path_
# Track the steps performed so they can be reversed.
revert_plan = list()
def add_source_file_group_to_plan(file_group_: FileGroup, source_: pathlib.Path):
primary_path_ = file_group_.primary_path
if primary_path_ not in plan:
new_primary_path_ = destination / source_.name / primary_path_.relative_to(source)
if new_primary_path_.exists():
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
plan[primary_path_] = new_primary_path_
def do_plan(plan_):
# Move deepest files first.
plan_ = sorted(plan_, key=lambda i: (len(i[0].parents), i[0].name), reverse=True)
for old_file, new_file in plan_: # noqa
new_file.parent.mkdir(parents=True, exist_ok=True)
logger.debug(f'Moving {old_file} to {new_file}')
# if not new_file.exists():
shutil.move(str(old_file), str(new_file))
logger.debug(f'Moved {old_file} to {new_file}')
revert_plan.append((new_file, old_file))
# Move is complete.
return plan_
# Sources may be a list of files, we only want to issue a query for one file in each FileGroup.
sources = get_unique_files_by_stem(sources)
logger.info(f'move got {len(sources)} to move')
try:
plan = do_plan(plan)
with flags.refresh_counting:
for source in sources:
if source.is_dir():
delete_directory(source)
logger.info(f'Move execution completed')
return plan
except Exception as e:
logger.error(f'Move failed', exc_info=e)
# Move files back.
do_plan(revert_plan)
# Delete the directories that were created.
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
for path in directories:
if path.is_dir():
delete_directory(path)
# Remove destination only if it did not exist at the start.
if destination.is_dir() and destination_existed is False:
delete_directory(destination)
raise
if source.is_file():
# Get any FileGroups that share the source's stem.
files = glob_shared_stem(source)
file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(files)).all()
if not file_groups:
# No FileGroups for this source file, create one.
try:
fg = FileGroup.from_paths(session, *files)
file_groups = [fg, ]
except NoPrimaryFile:
for file in files:
fg = FileGroup.from_paths(session, file)
add_file_group_to_plan(fg)
add_files_to_refresh_count(len(file_groups))
for fg in file_groups:
add_file_group_to_plan(fg)
# Sleep to catch cancel.
await asyncio.sleep(0)
elif source.is_dir():
stmt = 'SELECT primary_path FROM file_group WHERE primary_path LIKE :like'
for (primary_path,) in session.execute(stmt, dict(like=f'{source}%')):
primary_path = pathlib.Path(primary_path)
if primary_path not in plan:
new_primary_path = destination / source.name / primary_path.relative_to(source)
if new_primary_path.exists():
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path}')
plan[primary_path] = new_primary_path
add_files_to_refresh_count(1)
# Sleep to catch cancel.
await asyncio.sleep(0)
# Move directories of this source.
for directory in (i for i in walk(source) if i.is_dir()):
new_directory = destination / source.name / directory.relative_to(source)
plan[directory] = new_directory
# Find any files in this directory that are not yet in the DB and add them to the plan.
files = {i for i in walk(source) if i.is_file()}
if missing_files := files - set(plan.keys()):
for paths in group_files_by_stem(missing_files):
try:
# Only create the FileGroup if a primary_path can be found.
get_primary_file(paths)
file_group = FileGroup.from_paths(session, *paths)
add_files_to_refresh_count(1)
add_source_file_group_to_plan(file_group, source)
except NoPrimaryFile:
# No primary path for these paths, create FileGroups for each.
add_files_to_refresh_count(1)
for file in paths:
fg = FileGroup.from_paths(session, file)
add_source_file_group_to_plan(fg, source)
# Sleep to catch cancel.
await asyncio.sleep(0)
else:
raise UnknownFile(f'Unknown path type, cannot move: {source}')
# Sort plan by the deepest files first.
plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True))
# Revert plan is built out as files are moved.
revert_plan = OrderedDict()
new_directories = set()
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path]):
"""Apply the move plan to all the FileGroups.
@warning: Cannot be cancelled!"""
# Move FileGroups in groups.
for chunk in chunks(plan_.items(), MOVE_CHUNK_SIZE):
old_paths = [i for i, j in chunk]
old_files, old_directories = partition(lambda i: i.is_file(), old_paths)
file_groups_ = session.query(FileGroup).filter(FileGroup.primary_path.in_(old_files)).all()
if len(file_groups_) != len(old_files):
# We ensured there are FileGroups while building the plan, maybe user deleted a file?
raise RuntimeError('Could not get all FileGroups for move')
for file_group_ in file_groups_:
old_file = file_group_.primary_path
new_primary_path_ = plan_[old_file]
parent = new_primary_path_.parent
parent.mkdir(parents=True, exist_ok=True)
if parent not in new_directories:
new_directories.add(parent)
file_group_.move(new_primary_path_)
revert_plan[new_primary_path_] = old_file
for directory_ in old_directories:
delete_directory(directory_)
existing_directories = {i[0] for i in session.query(Directory.path)}
missing_directories = new_directories - existing_directories
for directory_ in missing_directories:
session.add(Directory(path=directory_, name=directory_.name))
# Don't forget what has moved.
session.flush(file_groups_)
with flags.refresh_discovery:
try:
do_plan(plan)
for source in sources:
if source.is_dir():
delete_directory(source)
logger.info(f'Move execution completed')
except Exception as e:
logger.error(f'Move failed', exc_info=e)
new_directories = set()
# Move files back. Get copy because do_plan will change the revert plan.
do_plan(revert_plan.copy())
# Delete the directories that were created.
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
for path in directories:
if path.is_dir():
delete_directory(path)
# Remove destination only if it did not exist at the start.
if destination.is_dir() and destination_existed is False:
delete_directory(destination)
raise
return plan
def delete_directory(directory: pathlib.Path, recursive: bool = False):
@ -1263,41 +1366,37 @@ def delete_directory(directory: pathlib.Path, recursive: bool = False):
curs.execute(stmt, (str(directory),))
async def move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
@optional_session
async def move(destination: pathlib.Path, *sources: pathlib.Path, session: Session = None) \
-> OrderedDict[pathlib.Path, pathlib.Path]:
"""Moves a file or directory (recursively), preserving applied Tags.
If the move fails at any point, the files will be returned to their previous locations.
Returns a list of tuples, the first Path in the tuple is the file's old location, the second Path is where the file
was moved.
Returns an OrderedDict, the key is the file's old location, the value is where the file was moved.
"""
# Move the files first, if this fails it will revert itself and raise an error.
plan = _move(destination, *sources)
if not session:
raise RuntimeError('`session` is required')
# Files were moved, migrate any Tags.
plan_map = {old: new for old, new in plan}
old_file_paths = list(plan_map.keys())
with get_db_session(commit=True) as session:
old_files: List[Tuple[FileGroup, TagFile]] = session.query(FileGroup, TagFile) \
.filter(FileGroup.primary_path.in_(old_file_paths)) \
.outerjoin(TagFile).all()
logger.info(f'Moving {len(old_files)} tagged files')
for old_file, _ in old_files:
try:
new_file_path = plan_map[old_file.primary_path]
logger.info(f'Moving tagged file {old_file.primary_path} to {new_file_path}')
old_file.move(new_file_path, move_files=False)
except KeyError as e:
# Got a duplicate?
logger.error('Error during tag migration', exc_info=e)
if PYTEST:
raise
session.commit()
with flags.refreshing:
from wrolpi.api_utils import api_app
api_app.shared_ctx.refresh['counted_files'] = 0
if PYTEST:
await refresh_files([destination, *sources])
else:
background_task(refresh_files([destination, *sources]))
# Move the files, if this fails it will revert itself and raise an error.
with timer('moving files', 'info'):
plan = await _move(destination, *sources, session=session)
session.commit()
with flags.refresh_indexing:
await apply_indexers()
with flags.refresh_modeling:
await apply_modelers()
with flags.refresh_cleanup:
await apply_refresh_cleanup()
# Save tags now that files have been moved.
save_tags_config()
return plan
@ -1320,25 +1419,40 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
return new_path
async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path:
async def rename_directory(directory: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) \
-> pathlib.Path:
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
if not session:
raise RuntimeError('`session` is required')
new_directory = directory.with_name(new_name)
if new_directory.exists():
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
# Move all paths into the new directory.
paths = list(directory.iterdir())
await move(new_directory, *paths)
try:
await move(new_directory, *paths, session=session)
if send_events:
Events.send_file_move_completed(f'Directory has been renamed: {directory}')
except Exception:
if send_events:
Events.send_file_move_failed(f'Directory rename has failed: {directory}')
raise
# Remove the old directory.
delete_directory(directory)
return new_directory
async def rename(path: pathlib.Path, new_name: str) -> pathlib.Path:
@optional_session
async def rename(path: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) -> pathlib.Path:
"""Rename a directory or file. Preserve any tags."""
if not session:
raise RuntimeError('`session` is required')
if path.is_dir():
return await rename_directory(path, new_name)
return await rename_directory(path, new_name, session=session, send_events=send_events)
return await rename_file(path, new_name)
@ -1399,26 +1513,36 @@ async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_nam
paths = glob_shared_stem(pathlib.Path(file))
# Remove any ignored files.
paths = remove_files_in_ignored_directories(paths)
if paths:
file_group = FileGroup.from_paths(session, *paths)
logger.debug(f'upsert_file: {file_group}')
# Re-index the contents of the file.
session.add(file_group)
session.flush([file_group, ])
try:
file_group.do_model(session)
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
for tag_name in (tag_names or []):
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(tag_name)
file_group.add_tag(tag.id)
session.commit()
return file_group
else:
if not paths:
logger.warning('upsert_file called, but all files are in ignored directories!')
return
for i in range(2):
file_group = FileGroup.from_paths(session, *paths)
# Re-index the contents of the file.
try:
session.flush([file_group, ])
break
except IntegrityError:
# Another process inserted this FileGroup.
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
session.rollback()
continue
else:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
logger.debug(f'upsert_file: {file_group}')
try:
file_group.do_model(session)
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
for tag_name in (tag_names or []):
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(tag_name)
file_group.add_tag(tag.id)
session.commit()
return file_group

View file

@ -10,7 +10,7 @@ from sqlalchemy import types
from sqlalchemy.orm import deferred, relationship, Session
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
get_relative_to_media_directory
get_relative_to_media_directory, unique_by_predicate
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
from wrolpi.db import optional_session
from wrolpi.errors import FileGroupIsTagged, UnknownFile
@ -154,9 +154,8 @@ class FileGroup(ModelHelper, Base):
"""Add all `paths` to this FileGroup.files."""
from wrolpi.files.lib import get_mimetype
new_files = list(self.files) if self.files else list()
for file in paths:
new_files.append(dict(path=file, mimetype=get_mimetype(file)))
self.files = new_files
new_files.extend([dict(path=path, mimetype=get_mimetype(path)) for path in paths])
self.files = unique_by_predicate(new_files, lambda i: i['path'])
def my_files(self, *mimetypes: str) -> List[dict]:
"""Return all files related to this group that match any of the provided mimetypes.
@ -225,7 +224,9 @@ class FileGroup(ModelHelper, Base):
if session := Session.object_session(self):
from wrolpi.downloader import download_manager
if self.url and add_to_skip_list is True:
if self.url and (download := download_manager.get_download(session, self.url)):
download.delete(add_to_skip_list)
elif self.url and add_to_skip_list is True:
download_manager.add_to_skip_list(self.url)
session.delete(self)
@ -254,25 +255,26 @@ class FileGroup(ModelHelper, Base):
@classmethod
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
"""Create a new FileGroup which contains the provided file paths."""
from wrolpi.files.lib import get_primary_file, get_mimetype, split_path_stem_and_suffix
from wrolpi.files.lib import get_primary_file, get_mimetype, get_unique_files_by_stem
unique_stems = {split_path_stem_and_suffix(i, full=True)[0] for i in paths}
unique_stems = get_unique_files_by_stem(paths)
if len(unique_stems) > 1:
raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem.')
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
logger.debug(f'FileGroup.from_paths: {len(existing_groups)}')
if len(existing_groups) == 0:
# These paths have not been used previously, create a new FileGroup.
file_group = FileGroup()
primary_path = get_primary_file(paths)
file_group.primary_path = primary_path
file_group.append_files(*paths)
file_group.mimetype = get_mimetype(file_group.primary_path)
session.add(file_group)
elif len(existing_groups) == 1:
# Found one FileGroup with these paths, no need to create a new FileGroup.
file_group = existing_groups[0]
primary_path = file_group.primary_path
file_group.append_files(*paths)
primary_path = get_primary_file(file_group.my_paths())
else:
# Multiple FileGroups contain these paths as primary.
primary_path = get_primary_file(paths)
@ -284,8 +286,11 @@ class FileGroup(ModelHelper, Base):
if not isinstance(primary_path, pathlib.Path):
raise ValueError('Cannot create FileGroup without a primary path.')
file_group.primary_path = primary_path
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
file_group.size = sum(i.stat().st_size for i in paths)
file_group.mimetype = get_mimetype(file_group.primary_path)
logger.debug(f'FileGroup.from_paths: {file_group}')
return file_group
@ -329,28 +334,44 @@ class FileGroup(ModelHelper, Base):
self.files = collected_files
def move(self, new_primary_path: pathlib.Path, move_files: bool = True):
def move(self, new_primary_path: pathlib.Path):
"""Move all files in this group to a new location."""
from wrolpi.files.lib import split_path_stem_and_suffix
if move_files and new_primary_path.exists():
from wrolpi.files.lib import split_path_stem_and_suffix, glob_shared_stem
if new_primary_path.exists():
raise FileExistsError(f'Cannot move {self} to {new_primary_path} because it already exists.')
# Adopt any new files when a FileGroup has multiple files.
if len(self.my_files()) > 1:
self.append_files(*glob_shared_stem(self.primary_path))
new_name, _ = split_path_stem_and_suffix(new_primary_path, full=True)
# Need a deepcopy because changes to self.files are ignored otherwise.
new_files = copy.deepcopy(self.files)
# Ensure that all destination files do not yet exist before move.
for file in new_files:
_, suffix = split_path_stem_and_suffix(file['path'])
new_path = pathlib.Path(f'{new_name}{suffix}')
if new_path.exists():
raise FileExistsError(f'Cannot move {self} to {new_path} because it already exists.')
# Collect all files that exist and move them.
existing_files = list()
for idx, file in enumerate(new_files):
_, suffix = split_path_stem_and_suffix(file['path'])
new_path = pathlib.Path(f'{new_name}{suffix}')
if move_files:
if pathlib.Path(file['path']).is_file():
shutil.move(file['path'], new_path)
new_files[idx]['path'] = new_path
self.files = new_files
existing_files.append({'path': new_path, 'mimetype': file['mimetype']})
logger.debug(f'Moved FileGroup: {self.primary_path} -> {new_primary_path}')
self.files = existing_files
if self.title == self.primary_path.name:
# Do not overwrite title from modeler.
self.title = new_primary_path.name
self.primary_path = new_primary_path
# Need to re-index for self.data.
self.indexed = False
self.indexed = False if self.data else self.indexed
# Flush the changes to the FileGroup.
self.flush()
@ -384,11 +405,15 @@ class FileGroup(ModelHelper, Base):
if model_class := self.get_model_class():
if model := model_class.get_by_path(self.primary_path, session):
# Already modeled.
logger.debug(f'already modeled: {self} {model}')
return model
# Create new model (Video/Archive/Ebook).
return model_class.do_model(self, session)
model = model_class.do_model(self, session)
logger.debug(f'new model: {self} {model}')
return model
# Index the file if no models are available.
logger.debug(f'no model class: {self}')
self.do_index()
self.indexed = True

View file

@ -190,7 +190,7 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi
@pytest.mark.asyncio
async def test_files_search_any_tag(test_async_client, test_session, make_files_structure, tag_factory):
one, two = tag_factory(), tag_factory()
one, two = await tag_factory(), await tag_factory()
files = [
'foo.txt',
'foo bar.txt',
@ -299,34 +299,35 @@ def test_file_statistics(test_session, test_client, test_directory, example_pdf,
}
def test_file_group_tag_by_primary_path(test_session, test_client, test_directory, example_singlefile, tag_factory,
insert_file_group):
@pytest.mark.asyncio
async def test_file_group_tag_by_primary_path(test_session, test_async_client, test_directory, example_singlefile,
tag_factory, insert_file_group):
singlefile = FileGroup.from_paths(test_session, example_singlefile)
tag1 = tag_factory()
tag2 = tag_factory()
tag1 = await tag_factory()
tag2 = await tag_factory()
test_session.commit()
# FileGroup can be tagged with its primary_path.
content = dict(file_group_primary_path=str(singlefile.primary_path.relative_to(test_directory)), tag_name=tag1.name)
request, response = test_client.post('/api/files/tag', content=json.dumps(content))
request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 1
# FileGroup can be tagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
request, response = test_client.post('/api/files/tag', content=json.dumps(content))
request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 2
# FileGroup can be untagged with its primary_path.
content = dict(file_group_id=singlefile.id, tag_id=tag1.id)
request, response = test_client.post('/api/files/untag', content=json.dumps(content))
request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 1
# FileGroup can be untagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
request, response = test_client.post('/api/files/untag', content=json.dumps(content))
request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 0
@ -468,11 +469,13 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f
assert test_session.query(FileGroup).count() == 1
def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form, tag_factory,
video_bytes):
@pytest.mark.asyncio
async def test_post_upload(test_session, test_async_client, test_directory, make_files_structure, make_multipart_form,
tag_factory,
video_bytes):
"""A file can be uploaded in chunks directly to the destination."""
make_files_structure(['uploads/'])
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
forms = [
@ -487,7 +490,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = test_client.post('/api/files/upload', content=body, headers=headers)
request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK, response.content.decode()
output = test_directory / 'uploads/video.mp4'
@ -505,7 +508,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
dict(name='chunk', value=part2, filename='chunk'),
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', content=body, headers=headers)
request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
assert output.is_file()

View file

@ -137,7 +137,7 @@ async def test_discover_calibre_cover(test_session, test_directory, example_epub
@pytest.mark.asyncio
async def test_move_ebook(test_async_client, test_session, test_directory, example_epub, image_file, tag_factory):
"""An ebook is re-indexed when moved."""
tag = tag_factory()
tag = await tag_factory()
shutil.move(image_file, example_epub.with_suffix('.jpg'))
await refresh_files()

View file

@ -82,7 +82,7 @@ async def test_delete_file_names(test_async_client, test_session, make_files_str
})
foo_fg = FileGroup.from_paths(test_session, foo)
foo1_fg = FileGroup.from_paths(test_session, foo1)
tag = tag_factory()
tag = await tag_factory()
foo1_fg.add_tag(tag.id)
test_session.commit()
assert foo.is_file()
@ -106,7 +106,7 @@ async def test_delete_file_link(test_async_client, test_session, test_directory)
@pytest.mark.asyncio
async def test_delete_tagged(test_async_client, test_session, make_files_structure, tag_factory, video_bytes):
"""Cannot delete a file that has been tagged."""
tag = tag_factory()
tag = await tag_factory()
make_files_structure({'foo/bar.txt': 'asdf', 'foo/bar.mp4': video_bytes})
await lib.refresh_files()
# Both files end up in a group.
@ -365,7 +365,7 @@ async def test_file_group_tag(test_session, make_files_structure, test_directory
"""A FileGroup can be tagged."""
make_files_structure(['foo.mp4'])
await lib.refresh_discover_paths([test_directory, ], now())
one = tag_factory()
one = await tag_factory()
foo: FileGroup = test_session.query(FileGroup).one()
foo.add_tag(one.id)
@ -401,7 +401,7 @@ async def test_refresh_many_files(test_async_client, test_session, make_files_st
@pytest.mark.asyncio
async def test_refresh_cancel(test_session, make_files_structure, test_directory):
async def test_refresh_cancel(test_async_client, test_session, make_files_structure, test_directory):
"""Refresh tasks can be canceled."""
# Creat a lot of files so the refresh will take too long.
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
@ -723,15 +723,16 @@ async def test_refresh_directories(test_session, test_directory, assert_director
assert_directories({'foo', 'baz'})
def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory,
video_bytes, srt_file3):
@pytest.mark.asyncio
async def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory,
video_bytes, srt_file3):
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
migrated."""
vid, srt = make_files_structure({
'vid.mp4': video_bytes,
'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(),
})
one, two = tag_factory(), tag_factory()
one, two = await tag_factory(), await tag_factory()
vid_group = FileGroup.from_paths(test_session, vid)
srt_group = FileGroup.from_paths(test_session, srt)
test_session.add_all([vid_group, srt_group])
@ -774,10 +775,11 @@ async def test_move(test_async_client, test_session, test_directory, make_files_
})
foo = test_directory / 'foo'
qux = test_directory / 'qux'
await lib.refresh_files()
# mv foo qux
plan = await lib.move(qux, foo)
plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan]
plan = await lib.move(qux, foo, session=test_session)
plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan.items()]
# The deepest files are moved first.
assert plan == [('foo/bar/baz/archive.html', 'qux/foo/bar/baz/archive.html'),
('foo/bar/video.mp4', 'qux/foo/bar/video.mp4'),
@ -805,10 +807,16 @@ async def test_move_files(test_async_client, test_session, test_directory, make_
})
dest = test_directory / 'dest'
await lib.move(dest, one, two)
plan = await lib.move(dest, one, two, session=test_session)
assert list(plan.items()) == [
(test_directory / 'foo/one.txt', test_directory / 'dest/one.txt'),
(test_directory / 'two.txt', test_directory / 'dest/two.txt'),
]
# one.txt is moved out of foo.
assert (dest / 'one.txt').read_text() == 'one'
assert (dest / 'two.txt').read_text() == 'two'
assert not (test_directory / 'foo/one.txt').exists(), 'one.txt is lingering in source'
assert not (test_directory / 'two.txt').exists(), 'two.txt is lingering in source'
assert (dest / 'one.txt').read_text() == 'one', 'one.txt was not moved to destination'
assert (dest / 'two.txt').read_text() == 'two', 'two.txt was not moved to destination'
@pytest.mark.asyncio
@ -830,8 +838,14 @@ async def test_move_deep_directory(test_async_client, test_session, test_directo
bar, qux = baz.parent, quux.parent
dest = test_directory / 'deep/dest'
# mv foo/bar foo/qux foo/quuz.mp4 deep/dest
await lib.move(dest, bar, qux, quuz_mp4)
# mv foo/bar foo/qux foo/quuz.mp4 foo/quuz.txt deep/dest
plan = await lib.move(dest, bar, qux, quuz_mp4, quuz_txt, session=test_session)
assert list(plan.items()) == [
(test_directory / 'foo/qux/quux.txt', test_directory / 'deep/dest/qux/quux.txt'),
(test_directory / 'foo/bar/baz.txt', test_directory / 'deep/dest/bar/baz.txt'),
(test_directory / 'foo/quuz.txt', test_directory / 'deep/dest/quuz.txt'),
(test_directory / 'foo/quuz.mp4', test_directory / 'deep/dest/quuz.mp4'),
]
# Unrelated files and directories are untouched.
assert not (test_directory / 'deep/dest/foo').is_dir(), 'foo/ should not have been moved.'
@ -861,7 +875,7 @@ async def test_move_directory(test_async_client, test_session, test_directory, m
assert_directories({'foo'})
bar = test_directory / 'bar'
await lib.rename(test_directory / 'foo', 'bar')
await lib.rename(test_directory / 'foo', 'bar', session=test_session)
assert (bar / 'one.txt').read_text() == 'one'
assert_directories({'bar'})
@ -890,7 +904,7 @@ async def test_move_error(test_session, test_directory, make_files_structure, vi
return shutil.move(*args, **kwargs)
with mock.patch('wrolpi.files.lib.shutil.move', mock_shutil_move), pytest.raises(FileNotFoundError):
await lib.move(qux, foo)
await lib.move(qux, foo, session=test_session)
# The move failed, the files should be moved back.
# foo was not deleted.
assert foo.is_dir()
@ -923,21 +937,19 @@ async def test_file_group_move(test_async_client, test_session, make_files_struc
assert new_srt.read_text() == srt_text
assert not video.is_file()
assert not srt.is_file()
# Moved files must be re-indexed.
assert file_group.indexed is False
@pytest.mark.asyncio
async def test_move_tagged(test_async_client, test_session, test_directory, make_files_structure, tag_factory):
"""A FileGroup's tag is preserved when moved or renamed."""
tag = tag_factory()
tag = await tag_factory()
foo, bar, = make_files_structure({
'foo/foo.txt': 'foo',
'foo/bar.txt': 'bar',
})
await lib.refresh_files()
bar_file_group, foo_file_group = test_session.query(FileGroup).order_by(FileGroup.primary_path)
foo_file_group.title = 'custom title' # Should not be overwritten.
bar_file_group.title = 'custom title' # Should not be overwritten.
bar_file_group.add_tag(tag.id)
test_session.commit()
@ -945,7 +957,7 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make
qux.mkdir()
# Move both files into qux. The Tag should also be moved.
await lib.move(qux, bar, foo)
await lib.move(qux, bar, foo, session=test_session)
new_foo = qux / 'foo.txt'
new_bar = qux / 'bar.txt'
# Files were moved.
@ -963,10 +975,10 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make
assert not foo_file_group.tag_files
assert foo_file_group.primary_path.is_file()
assert foo_file_group.primary_path == new_foo == foo_file_group.files[0]['path']
assert foo_file_group.title == 'custom title', 'Custom title should not have been overwritten.'
assert bar_file_group.title == 'custom title', 'Custom title should not have been overwritten.'
# Rename "bar.txt" to "baz.txt"
await lib.rename(new_bar, 'baz.txt')
await lib.rename(new_bar, 'baz.txt', session=test_session)
baz = new_bar.with_name('baz.txt')
assert baz.read_text() == 'bar'
@ -1111,3 +1123,21 @@ async def test_upsert_file_video_with_channel(test_async_client, test_session, t
assert video.channel_id == channel.id, "Videos should be assigned to the Channel their files are in"
assert video.video_path, 'Video file was upserted'
assert video.info_json_path, 'Info json file not found near video file.'
@pytest.mark.asyncio
async def test_move_many_files(test_async_client, test_session, test_directory, make_files_structure):
make_files_structure([f'foo/{i}.txt' for i in range(10_000)])
await lib.refresh_files()
foo = test_directory / 'foo'
bar = test_directory / 'bar'
with timer('test_move_many_files'):
# mv foo bar
await lib.move(bar, foo, session=test_session)
assert not foo.is_dir()
assert bar.is_dir()
assert (bar / 'foo').is_dir()
assert (bar / 'foo/0.txt').is_file()

View file

@ -0,0 +1,21 @@
import pytest
@pytest.mark.asyncio
async def test_file_group_move(test_async_client, test_session, test_directory, video_factory):
"""Any new or renamed files are adopted during FileGroup.move."""
video = video_factory(title='video', with_poster_ext='.webp')
test_session.commit()
# Move one of the FileGroup's files. It should be moved as well.
video.poster_path.rename(video.poster_path.with_suffix('.jpg'))
# Move the FilGroup.
new_primary_path = video.video_path.with_name('new name')
video.file_group.move(new_primary_path)
# New file is discovered and adopted, missing file is removed.
assert video.file_group.my_paths() == [
(test_directory / 'videos/NO CHANNEL/new name.jpg'),
(test_directory / 'videos/NO CHANNEL/new name.mp4'),
]

View file

@ -23,7 +23,7 @@ from wrolpi.admin import HotspotStatus
from wrolpi.api_utils import json_response, json_error_handler, api_app
from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \
set_global_log_level, get_relative_to_media_directory
set_global_log_level, get_relative_to_media_directory, search_other_estimates
from wrolpi.dates import now
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
@ -31,6 +31,7 @@ from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
from wrolpi.events import get_events, Events
from wrolpi.files import files_bp
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
from wrolpi.tags import Tag
from wrolpi.vars import DOCKERIZED, IS_RPI, IS_RPI4, IS_RPI5, API_HOST, API_PORT, API_WORKERS, API_DEBUG, \
API_ACCESS_LOG, truthy_arg, API_AUTO_RELOAD
from wrolpi.version import __version__
@ -118,10 +119,16 @@ async def index(_):
@openapi.description('Echo whatever is sent to this.')
@openapi.response(HTTPStatus.OK, schema.EchoResponse)
async def echo(request: Request):
try:
request_json = request.json
except Exception as e:
logger.error('Failed to parse request JSON', exc_info=e)
request_json = None
request_headers = dict(request.headers)
ret = dict(
form=request.form,
headers=dict(request.headers),
json=request.json,
headers=request_headers,
json=request_json,
method=request.method,
args=request.args,
)
@ -137,7 +144,7 @@ def get_settings(_: Request):
ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories]
settings = {
'archive_directory': config.archive_destination,
'archive_destination': config.archive_destination,
'download_manager_disabled': download_manager.is_disabled,
'download_manager_stopped': download_manager.is_stopped,
'download_on_startup': config.download_on_startup,
@ -150,15 +157,15 @@ def get_settings(_: Request):
'ignore_outdated_zims': config.ignore_outdated_zims,
'ignored_directories': ignored_directories,
'log_level': api_app.shared_ctx.log_level.value,
'map_directory': config.map_destination,
'map_destination': config.map_destination,
'nav_color': config.nav_color,
'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative.
'throttle_on_startup': config.throttle_on_startup,
'throttle_status': admin.throttle_status().name,
'version': __version__,
'videos_directory': config.videos_destination,
'videos_destination': config.videos_destination,
'wrol_mode': config.wrol_mode,
'zims_directory': config.zims_destination,
'zims_destination': config.zims_destination,
}
return json_response(settings)
@ -482,7 +489,7 @@ async def get_tags_request(_: Request):
summary='Create or update a Tag',
)
async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None):
tags.upsert_tag(body.name, body.color, tag_id)
await tags.upsert_tag(body.name, body.color, tag_id)
if tag_id:
return response.empty(HTTPStatus.OK)
else:
@ -491,7 +498,7 @@ async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None):
@api_bp.delete('/tag/<tag_id:int>')
async def delete_tag_request(_: Request, tag_id: int):
tags.delete_tag(tag_id)
Tag.find_by_id(tag_id).delete()
return response.empty()
@ -594,4 +601,13 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate
return json_response(ret)
@api_bp.post('/search_other_estimates')
@validate(json=schema.SearchOtherEstimateRequest)
async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstimateRequest):
"""Used by the Global search to suggest FileGroup count to the user."""
others = await search_other_estimates(body.tag_names)
ret = dict(others=others)
return json_response(ret)
api_app.error_handler.add(Exception, json_error_handler)

View file

@ -206,3 +206,8 @@ class SearchFileEstimateRequest:
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot search for any tag, and list of tags.')
@dataclass
class SearchOtherEstimateRequest:
tag_names: List[str] = field(default_factory=lambda: list())

View file

@ -54,7 +54,9 @@ class Tag(ModelHelper, Base):
color = Column(String)
tag_files: List[TagFile] = relationship('TagFile', back_populates='tag', cascade='all')
tag_zim_entries: List[TagFile] = relationship('TagZimEntry', back_populates='tag', cascade='all')
tag_zim_entries: List = relationship('TagZimEntry', back_populates='tag', cascade='all')
channels: List = relationship('Channel', primaryjoin='Tag.id==Channel.tag_id', back_populates='tag',
cascade='all')
def __repr__(self):
name = self.name
@ -148,27 +150,71 @@ class Tag(ModelHelper, Base):
def has_relations(self) -> bool:
"""Returns True if this Tag has been used with any FileGroups or Zim Entries."""
return bool(self.tag_files or self.tag_zim_entries)
return bool(any(self.tag_files) or any(self.tag_zim_entries) or any(self.channels))
def update_tag(self, name: str, color: str, session: Session):
async def update_tag(self, name: str, color: str | None, session: Session):
"""Change name/color of tag. Ensures safeness of new values, checks for conflicts."""
if ',' in name:
raise InvalidTag('Tag name cannot have comma')
if not is_valid_hex_color(color):
if color and not is_valid_hex_color(color):
raise InvalidTag('Tag color is invalid')
# Replace forward-slash (linux directories) with unicode Big Solidus (U+29F8)
# Replace forward-slash (linux directories) with Unicode Big Solidus (U+29F8)
name = name.replace('/', '⧸')
name = INVALID_FILE_CHARS.sub('', name)
if Tag.get_by_name(name, session):
if (other := self.get_by_name(name, session)) and other.id != self.id:
raise InvalidTag('Tag name already taken')
if flags.refreshing.is_set():
raise RefreshConflict('Refusing to rename tag while file refresh is in progress')
old_name = self.name
self.name = name
self.color = color
self.color = color or self.color
if Session.object_session(self):
self.flush()
from modules.videos.models import Channel
from modules.videos.errors import ChannelDirectoryConflict
async def _():
for channel in self.channels:
channel: Channel
possible_directory = channel.format_directory(old_name)
if channel.directory == possible_directory:
# Channel is in this Tag's old directory, move the Channel to the new directory.
new_directory = channel.format_directory(name)
try:
new_directory.mkdir(parents=True)
except FileExistsError:
raise ChannelDirectoryConflict(f'Channel directory already exists: {new_directory}')
# Move the files of the Channel.
await channel.move_channel(new_directory, session)
else:
msg = f"Not moving Channel because it is not in this Tag's old directory:" \
f" {channel} {possible_directory} {self}"
logger.warning(msg)
if PYTEST:
await _()
else:
background_task(_())
def delete(self):
"""Deletes this Tag, if it is unused.
@warning: Commits and saves configs"""
if self.has_relations():
raise UsedTag(f'Cannot delete {self.name} it is used')
session = Session.object_session(self)
session.delete(self)
session.commit()
save_tags_config(session)
sync_tags_directory(session)
class TagsConfig(ConfigFile):
@ -394,16 +440,17 @@ def get_tags() -> List[dict]:
@optional_session
@run_after(save_tags_config)
def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag:
async def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag:
if tag_id:
tag = Tag.find_by_id(tag_id, session)
tag.update_tag(name, color, session)
tag.flush()
await tag.update_tag(name, color, session)
else:
tag = Tag()
tag.update_tag(name, color, session)
await tag.update_tag(name, color, session)
session.add(tag)
tag.flush()
try:
session.commit()
except IntegrityError as e:
@ -411,26 +458,12 @@ def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = Non
session.rollback()
raise InvalidTag(f'Name already taken') from e
save_tags_config()
sync_tags_directory()
save_tags_config(session)
sync_tags_directory(session)
return tag
@optional_session
def delete_tag(tag_id: int, session: Session = None):
tag = Tag.find_by_id(tag_id, session)
if tag.has_relations():
raise UsedTag(f'Cannot delete {tag.name} it is used')
session.delete(tag)
session.commit()
save_tags_config()
sync_tags_directory()
@optional_session
def import_tags_config(session: Session = None):
"""Reads the Tags and TagFiles from the config file, upserts them in the DB."""

View file

@ -901,3 +901,18 @@ def test_html_screenshot(singlefile_contents_factory):
])
def test_is_valid_hex_color(color, expected):
assert common.is_valid_hex_color(color) == expected
@pytest.mark.parametrize('iterable,predicate,expected', [
((1, 2, 2, 3, 4), lambda i: i, (1, 2, 3, 4)),
([1, 1, 2, 3, 3, 3, 3, 4], None, [1, 2, 3, 4]),
([1, 1, 2, 2, 3, 4, 5, 6, 7, 8], lambda i: i % 2, [1, 2]),
(['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach'], lambda i: len(i), ['apple', 'banana', 'pear']),
(
[pathlib.Path('foo.txt'), pathlib.Path('foo.mp4'), pathlib.Path('bar.txt')],
lambda i: i.stem,
[pathlib.Path('foo.txt'), pathlib.Path('bar.txt')]
),
])
def test_unique_by_predicate(iterable, predicate, expected):
assert common.unique_by_predicate(iterable, predicate) == expected

View file

@ -76,7 +76,7 @@ async def test_create_downloads(test_async_client, test_session, test_download_m
@pytest.mark.asyncio
async def test_recreate_download(test_session, test_download_manager, test_downloader, tag_factory):
"""Settings are preserved when a Download is restarted."""
tag = tag_factory()
tag = await tag_factory()
settings = {'tag_names': [tag.name, ]}
# Create download with settings initially.

View file

@ -392,7 +392,7 @@ async def test_get_status(test_async_client, test_session):
@pytest.mark.asyncio
async def test_download_crud(test_session, test_async_client, test_download_manager_config, tag_factory):
"""Test creating once-downloads and recurring downloads."""
tag1, tag2 = tag_factory(), tag_factory()
tag1, tag2 = await tag_factory(), await tag_factory()
async def dispatch_downloads(*a, **kw):
pass
@ -567,7 +567,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_)
assert response_.status_code == HTTPStatus.OK
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
tag1, tag2, tag3 = await tag_factory(), await tag_factory(), await tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
@ -629,7 +629,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
async def test_search_file_estimates_any_tag(test_session, test_async_client, archive_factory, tag_factory,
video_factory):
"""File estimates can be filtered by any_tag."""
tag1 = tag_factory()
tag1 = await tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar bunny')
test_session.commit()
@ -706,6 +706,12 @@ def test_recursive_errors():
@pytest.mark.asyncio
async def test_settings_special_directories(test_async_client, test_config):
"""Maintainer can change special directories."""
request, response = await test_async_client.get('/api/settings')
assert response.status_code == HTTPStatus.OK
assert response.json['archive_destination'] == 'archive/%(domain)s'
assert response.json['map_destination'] == 'map'
assert response.json['videos_destination'] == 'videos/%(channel_tag)s/%(channel_name)s'
assert response.json['zims_destination'] == 'zims'
data = {'archive_destination': 'archives'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
@ -731,3 +737,23 @@ async def test_settings_special_directories(test_async_client, test_config):
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
assert get_wrolpi_config().archive_destination == 'archive/%(domain)s'
@pytest.mark.asyncio
async def test_search_other_estimates(test_async_client, test_session, channel_factory, tag_factory):
# Can search without Tags or Channels.
body = dict(tag_names=[])
request, response = await test_async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['others']['channel_count'] == 0, 'Only one Channel is tagged.'
tag = await tag_factory()
channel1 = channel_factory(tag_name=tag.name)
channel2 = channel_factory()
test_session.commit()
# One Channel is tagged.
body = dict(tag_names=[tag.name, ])
request, response = await test_async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'

View file

@ -6,7 +6,7 @@ import yaml
from wrolpi import tags
from wrolpi.common import is_hardlinked, walk
from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged
from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged, UsedTag
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile, Tag
@ -15,8 +15,8 @@ from wrolpi.tags import TagFile, Tag
@pytest.mark.asyncio
async def test_tags_file_group_json(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
"""The tags of a file are returned in its JSON."""
tag_one = tag_factory()
tag_two = tag_factory()
tag_one = await tag_factory()
tag_two = await tag_factory()
await files_lib.refresh_files()
file_group: FileGroup = test_session.query(FileGroup).one()
@ -37,7 +37,7 @@ async def test_tags_file_group_json(test_async_client, test_session, make_files_
@pytest.mark.asyncio
async def test_tags_model(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
"""Can get Tag using class methods."""
tag1 = tag_factory()
tag1 = await tag_factory()
assert tag1.__json__()
assert Tag.find_by_id(tag1.id)
@ -63,7 +63,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc
await files_lib.refresh_files()
video_group: FileGroup = test_session.query(FileGroup).one()
tag_one = tag_factory()
tag_one = await tag_factory()
assert len(tag_one.tag_files) == 0
Tag.tag_file_group(video_group.id, tag_one.id)
@ -71,7 +71,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc
assert len(tag_one.tag_files) == 1
assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ]
tag_two = tag_factory()
tag_two = await tag_factory()
Tag.tag_file_group(video_group.id, tag_two.id)
test_session.commit()
assert len(tag_one.tag_files) == 1
@ -95,8 +95,8 @@ async def test_tags_config_file(test_async_client, test_session, test_directory,
await files_lib.refresh_files()
pdf: FileGroup = FileGroup.get_by_path(example_pdf, test_session)
video: FileGroup = FileGroup.get_by_path(video_file, test_session)
tag1 = tag_factory()
tag2 = tag_factory()
tag1 = await tag_factory()
tag2 = await tag_factory()
test_session.commit()
tags.save_tags_config(test_session)
@ -172,11 +172,17 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta
dict(name='ガーデン', color='#000000', id=1, file_group_count=1, zim_entry_count=0)]
assert_tags_config(tags={'ガーデン': {'color': '#000000'}})
# Tag color can be changed
content = dict(name='ガーデン', color='#ffffff')
request, response = await test_async_client.post('/api/tag/1', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert Tag.find_by_name('ガーデン').color == '#ffffff'
# Conflicting names return an error.
content = dict(name='ガーデン', color='#111111')
request, response = await test_async_client.post('/api/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert_tags_config(tags={'ガーデン': {'color': '#000000'}})
assert_tags_config(tags={'ガーデン': {'color': '#ffffff'}})
# Cannot delete Tag that is used.
request, response = await test_async_client.delete('/api/tag/1')
@ -197,7 +203,7 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta
async def test_delete_tagged_file(test_session, example_pdf, tag_factory):
"""You cannot delete a FileGroup if it is tagged."""
await files_lib.refresh_files()
tag = tag_factory()
tag = await tag_factory()
pdf: FileGroup = test_session.query(FileGroup).one()
pdf.add_tag(tag.id)
@ -277,7 +283,7 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
from modules.zim import lib as zim_lib
# Create some tags, use tag1.
tag1, tag2, tag3, tag4 = tag_factory(), tag_factory(), tag_factory(), tag_factory()
tag1, tag2, tag3, tag4 = await tag_factory(), await tag_factory(), await tag_factory(), await tag_factory()
# use tag1 for FileGroup.
foo, = make_files_structure({'foo': 'text'})
foo = FileGroup.from_paths(test_session, foo)
@ -324,16 +330,13 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
@pytest.mark.asyncio
async def test_invalid_tag(test_async_client, test_session, test_directory):
with pytest.raises(InvalidTag):
tags.upsert_tag('cannot use comma: ,', '#fff')
await tags.upsert_tag('cannot use comma: ,', '#fff')
with pytest.raises(InvalidTag):
tags.upsert_tag('invalid color', 'foo')
await tags.upsert_tag('invalid color', 'foo')
with pytest.raises(UnknownTag):
tags.upsert_tag('Tag ID does not exist', '#fff', 1)
with pytest.raises(UnknownTag):
tags.delete_tag(1)
await tags.upsert_tag('Tag ID does not exist', '#fff', 1)
@pytest.mark.asyncio
@ -347,7 +350,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
(test_directory / 'tags/cannot be deleted').mkdir(parents=True)
(test_directory / 'tags/cannot be deleted/because of this file').touch()
tag1, tag2 = tag_factory('First Aid'), tag_factory('Special/name')
tag1, tag2 = await tag_factory('First Aid'), await tag_factory('Special/name')
vid1 = video_factory(with_video_file=test_directory / 'vid1.mp4', with_caption_file=True, with_info_json=True,
with_poster_ext='jpg')
vid2 = video_factory(with_video_file=test_directory / 'vid2.mp4', with_info_json=True)
@ -443,7 +446,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
async def test_update_tag(test_async_client, test_session, test_directory, video_factory, tag_factory):
"""Linked files in the Tag Directory are moved when the tag is renamed."""
# Create tagged Video.
tag = tag_factory()
tag = await tag_factory()
video_factory(title='video', tag_names=[tag.name])
test_session.commit()
@ -453,10 +456,33 @@ async def test_update_tag(test_async_client, test_session, test_directory, video
assert (test_directory / 'tags/one/video.mp4').is_file()
# Invalid characters are replaced during rename.
tags.upsert_tag('new/name%', tag.color, tag.id)
await tags.upsert_tag('new/name%', tag.color, tag.id)
# Video file was moved.
assert (test_directory / 'tags').is_dir()
assert (test_directory / 'videos/NO CHANNEL/video.mp4').is_file()
assert not (test_directory / 'tags/one/video.mp4').exists()
assert (test_directory / 'tags/new⧸name/video.mp4').is_file()
@pytest.mark.asyncio
async def test_tag_rename_with_channel(test_async_client, test_session, test_directory, video_factory, tag_factory,
channel_factory):
tag = await tag_factory()
channel = channel_factory(name='Channel Name', tag_name=tag.name)
video_factory(title='video', channel_id=channel.id)
test_session.commit()
assert (test_directory / 'tags').is_dir()
assert (test_directory / 'videos/one/Channel Name').is_dir()
assert (test_directory / 'videos/one/Channel Name/video.mp4').is_file()
await tag.update_tag('New Tag Name', None, session=test_session)
assert (test_directory / 'tags').is_dir()
assert (test_directory / 'videos/New Tag Name/Channel Name').is_dir()
assert (test_directory / 'videos/New Tag Name/Channel Name/video.mp4').is_file()
# Cannot delete Tag when used by Channel. This prevents the need to move Channel directories when Tags are deleted.
with pytest.raises(UsedTag):
tag.delete()