Making example label a tag.

Tags can be created and deleted in UI.

Adding tag_file using id to avoid session error.

Using UTC in test.

Matching tag_names style in test.

Tag icon is displayed on file rows.

Archives, Videos and Files can be filtered by their tags.

Tags can be added and removed from UI.

Tags are displayed in VideoPlayer.

The tags of a file are returned in it's JSON.

Video favorites are converted to Favorite tag on channel config import.

Updating tests to match frontend usage.

PDF creator and modification time are displayed.

Reworking file search to use Video.__json__ for complex video relations.

Archives can be viewed.

Removing ebook.title.

Reworking alembic migration to be an upgrade rather than a replace.

Skipping test on circleci.

Clearing out video caption index only if extract_subtitles if off.

Channel tests pass.

Some video tests pass.

Missing videos can be found.  A video can be downloaded.

Orphaned files can be found.

Videos can be tagged.

Videos can be searched.

Videos can be modeled.

Converting some tests to use FileGroup.primary_path.

Adding archive and domains to alembic migration.

Testing cascading Archive titles.

Archives now uses FileGroup.

archive.lib tests pass.

Archive is partially converted to FileGroup.

Calculating FileGroup.primary file on refresh, rather than on select.

wrolpi tests pass.

Using FilesType to convert between str/pathlib.Path for FileGroup.files.

Using list of files in FileGroup.files instead of dictionary.

Storing files in dedicated FileGroup.files to avoid overwriting them.

Adding comment explaining CASE.

Not overwriting FileGroup.data on refresh if no change has been detected.

Removing File model in favor of FileGroup.data.

Adding typing to Files methods.

Files are indexed using the primary mimetype of the FileGroup.

files test_lib tests pass.

Removing file_group_file in favor of simple one-to-many relationship.

PDF files are indexed during refresh.

Adding Tag and TagFile to tag files with specific tags.

Only iterating when full_stem_ids has contents.

Testing that FileGroups are related to their Files.

Files and file_groups are created and linked.

Bump webpack from 5.75.0 to 5.76.1 in /app (#160)

Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1.
- [Release notes](https://github.com/webpack/webpack/releases)
- [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1)

---
updated-dependencies:
- dependency-name: webpack
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
Roland 2023-03-24 08:58:18 -06:00
parent 9fbfc8c0a7
commit 5615ce542d
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
71 changed files with 4160 additions and 3161 deletions

View file

@ -0,0 +1,233 @@
"""File group.
Revision ID: 3b6918aeca4b
Revises: 930b9b7adb79
Create Date: 2023-02-27 14:52:46.943351
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '3b6918aeca4b'
down_revision = '930b9b7adb79'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DELETE FROM video') # noqa
session.execute('ALTER TABLE video DROP COLUMN caption_path')
session.execute('ALTER TABLE video DROP COLUMN censored')
session.execute('ALTER TABLE video DROP COLUMN favorite')
session.execute('ALTER TABLE video DROP COLUMN info_json_path')
session.execute('ALTER TABLE video DROP COLUMN modification_datetime')
session.execute('ALTER TABLE video DROP COLUMN poster_path')
session.execute('ALTER TABLE video DROP COLUMN size')
session.execute('ALTER TABLE video DROP COLUMN title')
session.execute('ALTER TABLE video DROP COLUMN validated')
session.execute('ALTER TABLE video DROP COLUMN video_path')
session.execute('DELETE FROM archive') # noqa
session.execute('ALTER TABLE archive DROP COLUMN readability_json_path')
session.execute('ALTER TABLE archive DROP COLUMN readability_path')
session.execute('ALTER TABLE archive DROP COLUMN readability_txt_path')
session.execute('ALTER TABLE archive DROP COLUMN screenshot_path')
session.execute('ALTER TABLE archive DROP COLUMN singlefile_path')
session.execute('ALTER TABLE archive DROP COLUMN title')
session.execute('ALTER TABLE archive DROP COLUMN validated')
session.execute('DELETE FROM ebook') # noqa
session.execute('ALTER TABLE ebook DROP COLUMN cover_path')
session.execute('ALTER TABLE ebook DROP COLUMN ebook_path')
session.execute('ALTER TABLE ebook DROP COLUMN title')
session.execute('DROP TABLE IF EXISTS file')
session.execute('''
CREATE TABLE file_group (
id BIGSERIAL PRIMARY KEY,
data JSON,
files JSON,
full_stem TEXT NOT NULL UNIQUE,
idempotency TIMESTAMP WITH TIME ZONE,
indexed BOOLEAN DEFAULT FALSE,
mimetype TEXT NOT NULL,
model TEXT,
modification_datetime TIMESTAMP WITH TIME ZONE,
primary_path TEXT NOT NULL UNIQUE,
size BIGINT,
title TEXT,
a_text TEXT,
b_text TEXT,
c_text TEXT,
d_text TEXT,
textsearch tsvector GENERATED ALWAYS AS (
setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
) STORED
)''')
session.execute('ALTER TABLE video ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
session.execute('ALTER TABLE ebook ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
session.execute('ALTER TABLE archive ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
session.execute('''
CREATE TABLE tag (
id SERIAL PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
color VARCHAR(7)
)''')
session.execute('''
CREATE TABLE tag_file (
tag_id INTEGER REFERENCES tag(id),
file_group_id BIGINT REFERENCES file_group(id),
created_at TIMESTAMP WITH TIME ZONE,
UNIQUE (tag_id, file_group_id)
)''')
session.execute('''
INSERT INTO tag (name, color) VALUES
('Automotive','#000000'),
('Clothing','#9858C5'),
('Comms','#ECD71C'),
('Cooking','#00A04A'),
('Electronics','#000000'),
('Favorite','#000000'),
('Firearms','#000000'),
('Fire Safety','#000000'),
('First Aid','#C40405'),
('Food','#2F8055'),
('Fuel','#000000'),
('Garden','#3E8B22'),
('Husbandry','#000000'),
('Hygiene','#E6E6E6'),
('Knots','#000000'),
('Light','#EEEEEE'),
('Maps','#333333'),
('Mechanical','#7A7A7A'),
('Medical','#000000'),
('Nuclear','#000000'),
('Power','#D837FF'),
('Radio','#000000'),
('Security','#333333'),
('Shelter','#A18A62'),
('Survival','#A18A62'),
('Tools','#333333'),
('Water','#291BC2')
''')
session.execute('UPDATE wrolpi_flag SET refresh_complete = false')
if not DOCKERIZED:
session.execute('ALTER TABLE public.file_group OWNER TO wrolpi')
session.execute('ALTER TABLE public.tag OWNER TO wrolpi')
session.execute('ALTER TABLE public.tag_file OWNER TO wrolpi')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP TABLE IF EXISTS tag_file')
session.execute('DROP TABLE IF EXISTS tag')
session.execute('ALTER TABLE video DROP COLUMN file_group_id')
session.execute('ALTER TABLE ebook DROP COLUMN file_group_id')
session.execute('ALTER TABLE archive DROP COLUMN file_group_id')
session.execute('DROP TABLE file_group')
session.execute('''
create table file
(
path text not null primary key,
idempotency timestamp with time zone,
indexed boolean default false,
mimetype text,
suffix text,
model text,
associated boolean default false,
modification_datetime timestamp with time zone,
size bigint,
title text,
a_text text,
b_text text,
c_text text,
d_text text,
textsearch tsvector generated always as ((
((setweight(to_tsvector('english'::regconfig, COALESCE(a_text, ''::text)), 'A'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, ''::text)), 'B'::"char")) ||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, ''::text)), 'C'::"char")) ||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, ''::text)), 'D'::"char"))) stored,
full_stem text
);
alter table file
owner to postgres;
create index file_mimetype_idx
on file (mimetype);
create index file_textsearch_idx
on file using gin (textsearch);
create index file_modification_datetime_idx
on file (modification_datetime);
create index file_size_idx
on file (size);
create index file_model_idx
on file (model);
create index file_idempotency_idx
on file (idempotency);
create index file_suffix_idx
on file (suffix);
create index file_full_stem_idx
on file (full_stem);
''')
session.execute('DELETE FROM video') # noqa
session.execute('ALTER TABLE video ADD COLUMN caption_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE video ADD COLUMN censored BOOLEAN DEFAULT FALSE')
session.execute('ALTER TABLE video ADD COLUMN favorite TIMESTAMPTZ')
session.execute('ALTER TABLE video ADD COLUMN info_json_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE video ADD COLUMN poster_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE video ADD COLUMN video_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE video ADD COLUMN modification_datetime TIMESTAMPTZ')
session.execute('ALTER TABLE video ADD COLUMN size BIGINT')
session.execute('ALTER TABLE video ADD COLUMN title TEXT')
session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE')
session.execute('DELETE FROM archive') # noqa
session.execute('ALTER TABLE archive ADD COLUMN readability_json_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE archive ADD COLUMN readability_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE archive ADD COLUMN readability_txt_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE archive ADD COLUMN screenshot_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE archive ADD COLUMN singlefile_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE archive ADD COLUMN title TEXT')
session.execute('ALTER TABLE archive ADD COLUMN validated BOOLEAN DEFAULT FALSE')
session.execute('DELETE FROM ebook') # noqa
session.execute('ALTER TABLE ebook ADD COLUMN cover_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE ebook ADD COLUMN ebook_path TEXT REFERENCES file(path)')
session.execute('ALTER TABLE ebook ADD COLUMN title TEXT')
session.execute('UPDATE wrolpi_flag SET refresh_complete = false')
if not DOCKERIZED:
session.execute('ALTER TABLE public.file OWNER TO wrolpi')

16
app/package-lock.json generated
View file

@ -15,6 +15,7 @@
"lodash": "^4.17.21",
"query-string": "^8.1.0",
"react": "^18.2.0",
"react-colorful": "^5.6.1",
"react-dom": "^18.2.0",
"react-dropzone": "^14.2.3",
"react-qr-code": "^2.0.11",
@ -14438,6 +14439,15 @@
"node": ">=14"
}
},
"node_modules/react-colorful": {
"version": "5.6.1",
"resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz",
"integrity": "sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==",
"peerDependencies": {
"react": ">=16.8.0",
"react-dom": ">=16.8.0"
}
},
"node_modules/react-dev-utils": {
"version": "12.0.1",
"resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-12.0.1.tgz",
@ -27847,6 +27857,12 @@
"whatwg-fetch": "^3.6.2"
}
},
"react-colorful": {
"version": "5.6.1",
"resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz",
"integrity": "sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==",
"requires": {}
},
"react-dev-utils": {
"version": "12.0.1",
"resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-12.0.1.tgz",

View file

@ -10,6 +10,7 @@
"lodash": "^4.17.21",
"query-string": "^8.1.0",
"react": "^18.2.0",
"react-colorful": "^5.6.1",
"react-dom": "^18.2.0",
"react-dropzone": "^14.2.3",
"react-qr-code": "^2.0.11",

View file

@ -19,6 +19,7 @@ import {Donate} from "./components/Donate";
import {useEventsInterval} from "./Events";
import {SemanticToastContainer} from "react-semantic-toasts-2";
import {FilePreviewWrapper} from "./components/FilePreview";
import {TagsContext, useTags} from "./Tags";
function PageNotFound() {
const {t} = useContext(ThemeContext);
@ -75,7 +76,8 @@ const router = createBrowserRouter(createRoutesFromElements(<Route
</Route>));
export default function App() {
const status = useStatusInterval();
const statusValue = useStatusInterval();
const tagsValue = useTags();
useEventsInterval();
return <ThemeWrapper>
@ -85,9 +87,11 @@ export default function App() {
{/* Toasts can be on any page. */}
<SemanticToastContainer position='top-right'/>
<MediaContextProvider>
<StatusContext.Provider value={status}>
<RouterProvider router={router}/>
</StatusContext.Provider>
<TagsContext.Provider value={tagsValue}>
<StatusContext.Provider value={statusValue}>
<RouterProvider router={router}/>
</StatusContext.Provider>
</TagsContext.Provider>
</MediaContextProvider>
</FilePreviewWrapper>
</ThemeWrapper>

View file

@ -12,6 +12,7 @@ import {Divider, Icon, Message} from "semantic-ui-react";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {refreshFiles} from "./api";
import _ from "lodash";
import {TagsSegment} from "./Tags";
function FlagsMessages({flags}) {
if (!flags) {
@ -62,7 +63,7 @@ function FlagsMessages({flags}) {
export function Dashboard() {
useTitle('Dashboard');
const {searchStr, setSearchStr} = useSearchFiles();
const {searchStr, setSearchStr, activeTags} = useSearchFiles();
const {status} = useContext(StatusContext);
const wrol_mode = status ? status['wrol_mode'] : null;
@ -74,13 +75,14 @@ export function Dashboard() {
// Only show dashboard parts if not searching.
let body;
if (searchStr) {
if (searchStr || activeTags) {
body = <FilesSearchView showLimit={true} showSelectButton={true}/>;
} else {
body = <>
{!wrol_mode && downloads}
{/* Hide Status when user is starting a download */}
{!downloadOpen && <DashboardStatus/>}
<TagsSegment/>
</>;
}

315
app/src/Tags.js Normal file
View file

@ -0,0 +1,315 @@
import React, {useEffect, useState} from "react";
import {addTag, deleteTag, getTags, postTag, removeTag} from "./api";
import {
Button as SButton,
Confirm,
Dimmer,
Divider,
Form,
FormField,
FormInput,
Grid,
Header,
Label,
Loader,
Loader as SLoader,
Modal,
} from "semantic-ui-react";
import {contrastingColor, HelpPopup} from "./components/Common";
import {Button, Segment} from "./components/Theme";
import _ from "lodash";
import {Link} from "react-router-dom";
import {HexColorPicker} from "react-colorful";
export const TagsContext = React.createContext({tags: [], fetchTags: null});
export function useTags() {
const [tags, setTags] = React.useState(null);
const [tagNames, setTagNames] = React.useState(null);
const fetchTags = async () => {
try {
const t = await getTags();
setTags(t);
setTagNames(t.map(i => i['name']));
} catch (e) {
setTags([]);
setTagNames([]);
console.error(e);
}
}
const NameToTagLabel = ({name, color, to, ...props}) => {
if (tags && tags.length) {
for (let i = 0; i < tags.length; i++) {
const tag = tags[i];
if (name === tag['name']) {
const textColor = contrastingColor(tag['color']);
const label = <Label
size='large'
style={{backgroundColor: tag['color'], color: textColor}}
className='clickable'
{...props} // onClick passed here.
>
{name}
</Label>;
return label;
}
}
}
// Could not find tag by name, or no tags have been fetched.
return <Label size='large'>{name}</Label>;
}
const TagsGroup = ({tagNames, onClick}) => {
if (!tagNames || tagNames.length === 0) {
return <React.Fragment/>;
}
return <Label.Group tag>
{tagNames.map(i => <NameToTagLabel key={i} name={i} onClick={() => onClick(i)}/>)}
</Label.Group>
}
const TagsLinkGroup = ({tagNames}) => {
if (!tagNames || tagNames.length === 0) {
return <React.Fragment/>;
}
return <Label.Group tag>
{tagNames.map(i =>
<Link key={i} to={`/?tag=${i}`} style={{margin: '0.3em'}}>
<NameToTagLabel name={i}/>
</Link>
)}
</Label.Group>
}
useEffect(() => {
setTags([]);
fetchTags();
}, []);
return {tags, tagNames, NameToTagLabel, TagsGroup, TagsLinkGroup, fetchTags}
}
function DeleteTagLabel({name, onConfirm}) {
const {NameToTagLabel} = React.useContext(TagsContext);
const [confirmOpen, setConfirmOpen] = useState(false);
return <div style={{marginTop: '1em'}}>
<SButton color='red' onClick={() => setConfirmOpen(true)} icon='close'/>
<Confirm
id={`confirm${name}`}
open={confirmOpen}
content={`Are you sure you want to delete: ${name}?`}
confirmButton='Delete'
onCancel={() => setConfirmOpen(false)}
onConfirm={async () => onConfirm(name)}
/>
<NameToTagLabel name={name}/>
<br/>
</div>
}
function DeleteTagModal() {
const {fetchTags, tagNames} = React.useContext(TagsContext);
const [open, setOpen] = useState(false);
const localDeleteTag = async (name) => {
await deleteTag(name);
if (fetchTags) {
await fetchTags();
}
}
const content = tagNames && tagNames.length ?
tagNames.map(i => <DeleteTagLabel key={i} name={i} onConfirm={localDeleteTag}/>)
: <Dimmer><Loader inline active/></Dimmer>;
return <>
<Modal closeIcon
open={open}
onOpen={() => setOpen(true)}
onClose={() => setOpen(false)}
>
<Modal.Header>Delete Tags</Modal.Header>
<Modal.Content>
{content}
</Modal.Content>
</Modal>
<SButton color='red' onClick={() => setOpen(true)}>
Delete
</SButton>
</>
}
function CreateTagModal() {
const {fetchTags} = React.useContext(TagsContext);
const [open, setOpen] = useState(false);
const [tagName, setTagName] = useState('');
const [tagColor, setTagColor] = useState('#000000');
const textColor = contrastingColor(tagColor);
const saveTag = async () => {
await postTag(tagName, tagColor);
if (fetchTags) {
await fetchTags();
}
setTagName('');
}
return <>
<Modal closeIcon
open={open}
onOpen={() => setOpen(true)}
onClose={() => setOpen(false)}
>
<Modal.Header>Create New Tag</Modal.Header>
<Modal.Content>
<Label.Group tag>
<Label size='large' style={{backgroundColor: tagColor, color: textColor}}>
{tagName || 'Example Tag'}
</Label>
</Label.Group>
<Divider/>
<Form autoComplete='off'>
<FormField>
<FormInput required
label='Tag Name'
type='text'
placeholder='Unique Name'
value={tagName}
onChange={(e, {value}) => setTagName(value)}
/>
</FormField>
<HexColorPicker color={tagColor} onChange={setTagColor}/>
<SButton primary
size='big'
onClick={saveTag}
style={{marginTop: '2em'}}
disabled={!!!tagName}
>
Save
</SButton>
</Form>
</Modal.Content>
</Modal>
<SButton primary onClick={() => setOpen(true)}>New</SButton>
</>
}
export function TagsModal({fileGroup, onClick}) {
const {tags: usedTags} = fileGroup;
const {TagsGroup} = useTags();
const {tags: availableTags} = React.useContext(TagsContext);
const [open, setOpen] = useState(false);
const [loading, setLoading] = useState(false);
const localAddTag = async (name) => {
setLoading(true);
try {
await addTag(fileGroup, name);
await onClick();
console.debug('Added tag');
} finally {
setLoading(false);
}
};
const localRemoveTag = async (name) => {
setLoading(true);
try {
await removeTag(fileGroup, name);
await onClick();
console.debug('Removed tag');
} finally {
setLoading(false);
}
};
const availableTagNames = availableTags && availableTags.length ? availableTags.map(i => i['name']) : [];
const unusedTags = _.difference(availableTagNames, usedTags);
let availableTagsGroup = <SLoader active inline/>;
if (unusedTags && unusedTags.length) {
availableTagsGroup = <TagsGroup tagNames={unusedTags} onClick={localAddTag}/>;
}
let usedTagsGroup = 'Add tags by clicking them below';
if (usedTags && usedTags.length) {
// TODO sort these labels.
usedTagsGroup = <TagsGroup tagNames={usedTags} onClick={localRemoveTag}/>;
}
const modal = <Modal closeIcon
open={open}
onOpen={() => setOpen(true)}
onClose={() => setOpen(false)}>
<Modal.Content>
{loading && <Dimmer active><SLoader/></Dimmer>}
<Header as='h4'>Used Tags</Header>
{usedTagsGroup}
<Divider/>
{availableTagsGroup}
</Modal.Content>
<Modal.Actions>
<DeleteTagModal/>
<CreateTagModal/>
</Modal.Actions>
</Modal>
return <>
<Button icon='tag' onClick={() => setOpen(true)}/>
{modal}
</>
}
export const taggedImageLabel = {corner: 'left', icon: 'tag', color: 'green'};
export const TagsDisplay = ({fileGroup, onClick}) => {
const {TagsLinkGroup} = useTags();
return <Grid>
<Grid.Row>
<Grid.Column mobile={2} computer={1}>
<TagsModal fileGroup={fileGroup} onClick={onClick}/>
</Grid.Column>
<Grid.Column mobile={13} computer={14}>
<TagsLinkGroup tagNames={fileGroup['tags']}/>
{_.isEmpty(fileGroup['tags']) &&
<HelpPopup content='Click the button to add tags' iconSize={null}/>
}
</Grid.Column>
</Grid.Row>
</Grid>
}
export const TagsSegment = () => {
const {tagNames, TagsLinkGroup} = useTags();
let availableTagsGroup = <SLoader active inline/>;
if (tagNames && tagNames.length) {
availableTagsGroup = <TagsLinkGroup tagNames={tagNames}/>;
}
return <Segment>
<Header as='h2'>Tags</Header>
{availableTagsGroup}
<Divider/>
<DeleteTagModal/>
<CreateTagModal/>
</Segment>
}

View file

@ -103,11 +103,11 @@ export async function getChannel(id) {
return (await response.json())['channel'];
}
export async function searchVideos(offset, limit, channelId, searchStr, order_by, filters = []) {
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tags) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
let body = {offset, limit, filters, order_by: order_by || 'rank'};
let body = {offset, limit, order_by: order_by || 'rank'};
if (searchStr) {
body['search_str'] = searchStr;
@ -115,13 +115,16 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
if (channelId) {
body['channel_id'] = parseInt(channelId);
}
if (tags) {
body['tag_names'] = tags.split(',');
}
console.debug('searching videos', body);
let response = await apiPost(`${VIDEOS_API}/search`, body);
if (response.status === 200) {
let data = await response.json();
return [data['files'], data['totals']['files']];
return [data['file_groups'], data['totals']['file_groups']];
} else {
toast({
type: 'error',
@ -136,7 +139,7 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
export async function getVideo(video_id) {
let response = await apiGet(`${VIDEOS_API}/video/${video_id}`);
let data = await response.json();
return [data['file'], data['prev'], data['next']];
return [data['file_group'], data['prev'], data['next']];
}
export async function deleteVideos(videoIds) {
@ -203,12 +206,6 @@ export async function validateRegex(regex) {
return (await response.json())['valid'];
}
export async function favoriteVideo(video_id, favorite) {
let body = {favorite: favorite, video_id};
let response = await apiPost(`${VIDEOS_API}/favorite`, body);
return (await response.json())['favorite'];
}
export async function getVideosStatistics() {
let response = await apiGet(`${VIDEOS_API}/statistics`);
if (response.status === 200) {
@ -353,7 +350,7 @@ export async function deleteArchives(archiveIds) {
}
}
export async function searchArchives(offset, limit, domain, searchStr, order) {
export async function searchArchives(offset, limit, domain, searchStr, order, tags) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@ -367,12 +364,15 @@ export async function searchArchives(offset, limit, domain, searchStr, order) {
if (order) {
body['order_by'] = order;
}
if (tags) {
body['tag_names'] = tags.split(',');
}
console.debug('searching archives', body);
let response = await apiPost(`${ARCHIVES_API}/search`, body);
if (response.status === 200) {
let data = await response.json();
return [data['archives'], data['totals']['archives']];
return [data['file_groups'], data['totals']['file_groups']];
} else {
toast({
type: 'error',
@ -380,6 +380,7 @@ export async function searchArchives(offset, limit, domain, searchStr, order) {
description: 'Cannot search archives. See server logs.',
time: 5000,
});
return [[], 0];
}
}
@ -402,7 +403,7 @@ export async function getArchive(archiveId) {
const response = await apiGet(`${ARCHIVES_API}/${archiveId}`);
if (response.status === 200) {
const data = await response.json();
return [data['file'], data['history']];
return [data['file_group'], data['history']];
} else {
toast({
type: 'error', title: 'Archive Error', description: 'Unable to get Archive. See server logs.', time: 5000,
@ -465,17 +466,7 @@ export async function deleteDownload(downloadId) {
}
}
const replaceFileDatetimes = (files) => {
for (let i = 0; i < files.length; i++) {
let file = files[i];
if (file['modified']) {
files[i]['modified'] = file['modified'] * 1000;
}
}
return files;
}
export async function filesSearch(offset, limit, searchStr, mimetypes, model) {
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tags) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
if (mimetypes) {
body['mimetypes'] = mimetypes;
@ -483,14 +474,16 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model) {
if (model) {
body['model'] = model;
}
if (tags) {
body['tag_names'] = tags.split(',');
}
console.debug('searching files', body);
const response = await apiPost(`${API_URI}/files/search`, body);
if (response.status === 200) {
let data = await response.json();
let [files, total] = [data['files'], data['totals']['files']];
files = replaceFileDatetimes(files);
return [files, total];
let [file_groups, total] = [data['file_groups'], data['totals']['file_groups']];
return [file_groups, total];
} else {
toast({
type: 'error',
@ -507,9 +500,9 @@ export async function refreshFiles() {
}
export async function refreshDirectoryFiles(directory) {
let body = {directory};
let body = {paths: [directory]};
try {
return await apiPost(`${API_URI}/files/refresh/directory`, body);
return await apiPost(`${API_URI}/files/refresh`, body);
} catch (e) {
console.error(e);
toast({
@ -526,7 +519,6 @@ export async function getFiles(directories) {
let body = {directories: directories || []};
let response = await apiPost(`${API_URI}/files`, body);
let {files} = await response.json();
files = replaceFileDatetimes(files);
return files;
}
@ -676,3 +668,67 @@ export async function getEvents(after) {
return await response.json();
}
}
export async function getTags() {
let uri = `${API_URI}/tag`;
let response = await apiGet(uri);
if (response.status === 200) {
const body = await response.json();
return body['tags'];
}
}
export async function addTag(fileGroup, name) {
const {id} = fileGroup;
const uri = `${API_URI}/files/tag`;
const body = {file_group_id: id, tag_name: name};
let response = await apiPost(uri, body);
if (response.status !== 201) {
console.error('Failed to add tag');
}
}
export async function removeTag(fileGroup, name) {
const {id} = fileGroup;
const uri = `${API_URI}/files/untag`;
const body = {file_group_id: id, tag_name: name};
let response = await apiPost(uri, body)
if (response.status !== 204) {
console.error('Failed to remove tag');
}
}
export async function postTag(name, color) {
const body = {name: name, color: color};
const uri = `${API_URI}/tag/new`;
let response = await apiPost(uri, body);
if (response.status !== 201) {
console.error('Failed to create new tag');
toast({
type: 'error', title: 'Error!', description: 'Unable to save tag', time: 5000,
})
} else {
toast({
type: 'info', title: 'Created new tag', description: `Created new tag: ${name}`, time: 2000,
});
}
}
export async function deleteTag(name) {
const body = {name: name};
const uri = `${API_URI}/tag/delete`;
let response = await apiPost(uri, body);
if (response.status === 400) {
const content = await response.json();
if (content['code'] === 38) {
toast({
type: 'error', title: 'Error!', description: content['message'], time: 5000,
})
}
} else if (response.status !== 204) {
console.error('Failed to delete tag');
toast({
type: 'error', title: 'Error!', description: `Unable to delete tag: ${name}`, time: 5000,
})
}
}

View file

@ -34,20 +34,21 @@ import {deleteArchives, postDownload} from "../api";
import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useArchive, useDomains, useQuery, useSearchArchives} from "../hooks/customHooks";
import {FileCards, FilesView} from "./Files";
import {FileCards, FileRowTagIcon, FilesView} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Dropdown from "semantic-ui-react/dist/commonjs/modules/Dropdown";
import _ from "lodash";
import {ThemeContext} from "../contexts/contexts";
import {Button, Card, CardIcon, Header, Loader, Placeholder, Segment} from "./Theme";
import {SortableTable} from "./SortableTable";
import {taggedImageLabel, TagsDisplay} from "../Tags";
function ArchivePage() {
const [deleteOpen, setDeleteOpen] = useState(false);
const [updateOpen, setUpdateOpen] = useState(false);
const navigate = useNavigate();
const {archiveId} = useParams();
const {archiveFile, history} = useArchive(archiveId);
const {archiveFile, history, fetchArchive} = useArchive(archiveId);
let title;
if (archiveFile && archiveFile.archive) {
@ -65,11 +66,11 @@ function ArchivePage() {
</>
}
const {archive} = archiveFile;
const {data} = archiveFile;
const singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null;
const screenshotUrl = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
const readabilityUrl = archive.readability_path;
const singlefileUrl = data.singlefile_path ? `/media/${encodeURIComponent(data.singlefile_path)}` : null;
const screenshotUrl = data.screenshot_path ? `/media/${encodeURIComponent(data.screenshot_path)}` : null;
const readabilityUrl = data.readability_path;
const singlefileButton = <ExternalCardLink to={singlefileUrl}>
<Button content='View' color='blue'/>
@ -85,12 +86,12 @@ function ArchivePage() {
const localDeleteArchive = async () => {
setDeleteOpen(false);
await deleteArchives([archive.id]);
await deleteArchives([data.id]);
navigate(-1);
}
const localUpdateArchive = async () => {
setUpdateOpen(false);
await postDownload(archive.url, 'archive');
await postDownload(data.url, 'archive');
}
const updateButton = (<>
@ -121,7 +122,7 @@ function ArchivePage() {
historyList = <FileCards files={history}/>;
}
const domain = archive.domain ? archive.domain.domain : null;
const domain = data.domain ? data.domain.domain : null;
let domainHeader;
if (domain) {
const domainUrl = `/archive?domain=${domain}`;
@ -133,10 +134,10 @@ function ArchivePage() {
}
let urlHeader;
if (archive.url) {
if (data.url) {
urlHeader = <Header as='h5'>
<ExternalCardLink to={archive.url}>
{textEllipsis(archive.url)}
<ExternalCardLink to={data.url}>
{textEllipsis(data.url)}
</ExternalCardLink>
</Header>;
}
@ -147,9 +148,9 @@ function ArchivePage() {
<Segment>
{screenshot}
<ExternalCardLink to={singlefileUrl}>
<Header as='h2'>{textEllipsis(archive.title || archive.url)}</Header>
<Header as='h2'>{textEllipsis(data.title || data.url)}</Header>
</ExternalCardLink>
<Header as='h3'>{isoDatetimeToString(archive.archive_datetime)}</Header>
<Header as='h3'>{isoDatetimeToString(data.archive_datetime)}</Header>
{domainHeader}
{urlHeader}
@ -159,6 +160,10 @@ function ArchivePage() {
{deleteButton}
</Segment>
<Segment>
<TagsDisplay fileGroup={archiveFile} onClick={fetchArchive}/>
</Segment>
<Segment>
<HelpHeader
headerContent='History'
@ -171,17 +176,18 @@ function ArchivePage() {
export function ArchiveCard({file}) {
const {s} = useContext(ThemeContext);
const {archive} = file;
const {data} = file;
const imageSrc = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
const singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null;
const imageSrc = data.screenshot_path ? `/media/${encodeURIComponent(data.screenshot_path)}` : null;
const singlefileUrl = data.singlefile_path ? `/media/${encodeURIComponent(data.singlefile_path)}` : null;
let screenshot = <CardIcon><FileIcon file={file}/></CardIcon>;
const imageLabel = file.tags && file.tags.length ? taggedImageLabel : null;
if (imageSrc) {
screenshot = <Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>;
screenshot = <Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}} label={imageLabel}/>;
}
const domain = archive.domain ? archive.domain.domain : null;
const domain = data ? data.domain : null;
const domainUrl = `/archive?domain=${domain}`;
return <Card color={mimetypeColor(file.mimetype)}>
@ -194,7 +200,7 @@ export function ArchiveCard({file}) {
<CardHeader>
<Container textAlign='left'>
<ExternalCardLink to={singlefileUrl}>
{cardTitleWrapper(archive.title || archive.url)}
{cardTitleWrapper(file.title || data.url)}
</ExternalCardLink>
</Container>
</CardHeader>
@ -204,15 +210,15 @@ export function ArchiveCard({file}) {
</CardLink>}
<CardMeta {...s}>
<p>
{isoDatetimeToString(archive.archive_datetime)}
{isoDatetimeToString(data.archive_datetime)}
</p>
</CardMeta>
<CardDescription>
<Link to={`/archive/${archive.id}`}>
<Link to={`/archive/${data.id}`}>
<Button icon='file alternate' content='Details'
labelPosition='left'/>
</Link>
<Button icon='external' href={archive.url} target='_blank' rel='noopener noreferrer'/>
<Button icon='external' href={data.url} target='_blank' rel='noopener noreferrer'/>
</CardDescription>
</CardContent>
</Card>
@ -427,10 +433,10 @@ function Archives() {
}
export function ArchiveRowCells({file}) {
const {archive} = file;
const {data} = file;
const archiveUrl = `/archive/${archive.id}`;
const posterUrl = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
const archiveUrl = `/archive/${data.id}`;
const posterUrl = data.screenshot_path ? `/media/${encodeURIComponent(data.screenshot_path)}` : null;
let poster;
if (posterUrl) {
@ -448,7 +454,8 @@ export function ArchiveRowCells({file}) {
</TableCell>
<TableCell>
<CardLink to={archiveUrl}>
{textEllipsis(archive.title || archive.stem)}
<FileRowTagIcon file={file}/>
{textEllipsis(file.title || file.stem)}
</CardLink>
</TableCell>
</React.Fragment>

View file

@ -1,5 +1,5 @@
import React, {useContext, useEffect, useState} from "react";
import {Card, Container, IconGroup, Input, Modal, Pagination} from 'semantic-ui-react';
import {Card, Container, IconGroup, Image, Input, Modal, Pagination} from 'semantic-ui-react';
import {Link, NavLink, useNavigate} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useDirectories, useDownloaders, useHotspot, useSettings, useThrottle} from "../hooks/customHooks";
@ -117,7 +117,7 @@ export function Duration({video}) {
export function isoDatetimeToString(dt) {
// Convert a datetime to a human-readable date format.
let d = <></>;
let d = <React.Fragment/>;
if (dt) {
d = new Date(dt);
d = `${d.getFullYear()}-${d.getMonth() + 1}-${d.getDate()}`;
@ -444,8 +444,8 @@ export function CardGroupCentered(props) {
export function CardPosterLink({to, poster_url, imageLabel, external = false}) {
const {s} = useContext(ThemeContext);
const style = {display: 'flex', justifyContent: 'center', ...s['style']};
const image = <img src={poster_url} alt={imageLabel}
style={{maxHeight: '163px', width: 'auto'}}
const image = <Image src={poster_url} label={imageLabel}
style={{maxHeight: '163px', width: 'auto'}}
/>;
if (external === true && to) {
return <a href={to} target='_blank' rel='noreferrer' style={style}>
@ -461,12 +461,12 @@ export function CardPosterLink({to, poster_url, imageLabel, external = false}) {
}
}
export function HelpPopup({icon, size, content, position}) {
export function HelpPopup({icon, size, content, position, iconSize = 'small'}) {
return <Popup
content={content}
size={size || null}
position={position || 'left center'}
trigger={<Icon circular link name={icon || 'question'} size='small'
trigger={<Icon circular link name={icon || 'question'} size={iconSize}
style={{marginLeft: '0.25em', marginRight: '0.25em'}}
/>}
/>
@ -616,8 +616,9 @@ export function isZipMimetype(mimetype) {
export function FileIcon({file, disabled = true, size = 'huge', ...props}) {
// Default to a grey file icon.
const {mimetype, path} = file;
const lowerPath = path.toLowerCase();
const {mimetype, path, primary_path} = file;
// `file` may be a file_group or a file.
const lowerPath = primary_path ? primary_path.toLocaleString() : path.toLowerCase();
props['name'] = 'file';
props['color'] = mimetypeColor(mimetype);
if (mimetype) {
@ -851,3 +852,25 @@ export const cardTitleWrapper = (title, maxLength = 100, breakWord = true) => {
const style = breakWord ? {overflowWrap: 'break-word'} : null;
return <span style={style}>{textEllipsis(title, maxLength)}</span>
}
function luma(color) {
let rgb = (typeof color === 'string') ? hexToRGBArray(color) : color;
return (0.2126 * rgb[0]) + (0.7152 * rgb[1]) + (0.0722 * rgb[2]); // SMPTE C, Rec. 709 weightings
}
function hexToRGBArray(color) {
if (color.length === 7) {
color = color.slice(1);
}
if (color.length !== 6) {
console.error('Invalid hex color: ' + color);
return;
}
let rgb = [];
for (let i = 0; i <= 2; i++) rgb[i] = parseInt(color.substr(i * 2, 2), 16);
return rgb;
}
export function contrastingColor(color) {
return (luma(color) >= 120) ? '#000000' : '#dddddd';
}

View file

@ -7,6 +7,7 @@ import {
Container,
Dropdown,
Image,
Label,
PlaceholderLine,
TableCell,
TableHeaderCell,
@ -40,6 +41,7 @@ import {FileBrowser} from "./FileBrowser";
import {refreshDirectoryFiles, refreshFiles} from "../api";
import {useSubscribeEventName} from "../Events";
import {FilePreviewContext} from "./FilePreview";
import {TagsContext} from "../Tags";
export function FilesPage() {
@ -50,20 +52,18 @@ export function FilesPage() {
function EbookCard({file}) {
const {s} = useContext(ThemeContext);
let {ebook, suffix} = file;
let {data} = file;
const downloadUrl = `/download/${encodeURIComponent(file.path)}`;
const downloadUrl = `/download/${encodeURIComponent(file.primary_path)}`;
const isEpub = file['mimetype'].startsWith('application/epub');
const viewerUrl = isEpub ? `/epub.html?url=${downloadUrl}` : null;
let cover = <FileIcon file={file}/>;
if (ebook && ebook.cover_path) {
const coverSrc = `/media/${encodeURIComponent(ebook.cover_path)}`;
if (data && data.cover_path) {
const coverSrc = `/media/${encodeURIComponent(data.cover_path)}`;
cover = <CardPosterLink poster_url={coverSrc}/>;
}
suffix = suffix ? _.trimStart(suffix, '.').toUpperCase() : null;
const color = mimetypeColor(file.mimetype);
return <Card color={color}>
<PreviewLink file={file}>
@ -73,12 +73,13 @@ function EbookCard({file}) {
<CardHeader>
<Container textAlign='left'>
<ExternalCardLink to={viewerUrl || downloadUrl}>
{ebook ? ebook.title : file.title}
{data ? data.title : file.title}
</ExternalCardLink>
</Container>
</CardHeader>
<CardMeta>
<pre {...s}>{suffix}</pre>
{data.creator ? <b {...s}>{data.creator}</b> : null}
{file.size && <p {...s}>{humanFileSize(file.size)}</p>}
</CardMeta>
</CardContent>
</Card>
@ -88,7 +89,7 @@ function EbookCard({file}) {
function ImageCard({file}) {
const {s} = useContext(ThemeContext);
const {setPreviewFile} = React.useContext(FilePreviewContext);
const url = `/media/${encodeURIComponent(file.path)}`;
const url = `/media/${encodeURIComponent(file.primary_path)}`;
let poster = <FileIcon file={file}/>;
if (file.size && file.size < 50000000) {
@ -107,7 +108,7 @@ function ImageCard({file}) {
<CardContent {...s}>
<CardHeader>
<ExternalCardLink to={url} className='no-link-underscore card-link'>
<p>{textEllipsis(file.title || file.stem || file.path)}</p>
<p>{textEllipsis(file.title || file.stem || file.primary_path)}</p>
</ExternalCardLink>
</CardHeader>
<CardMeta {...s}>
@ -127,17 +128,19 @@ function FileCard({file}) {
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
);
if (file.model === 'video' && file['video']) {
return <VideoCard key={file['path']} file={file}/>;
} else if (file.model === 'archive' && file['archive']) {
return <ArchiveCard key={file['path']} file={file}/>;
if (file.model === 'video') {
return <VideoCard key={file['primary_path']} file={file}/>;
} else if (file.model === 'archive') {
return <ArchiveCard key={file['primary_path']} file={file}/>;
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
return <ImageCard key={file['path']} file={file}/>;
return <ImageCard key={file['primary_path']} file={file}/>;
} else if (isEbookType) {
return <EbookCard key={file['path']} file={file}/>;
return <EbookCard key={file['primary_path']} file={file}/>;
}
const downloadUrl = `/download/${encodeURIComponent(file.path)}`;
const {data} = file;
const {author} = data;
const downloadUrl = `/download/${encodeURIComponent(file.primary_path)}`;
const color = mimetypeColor(file.mimetype);
const size = file.size !== null && file.size !== undefined ? humanFileSize(file.size) : null;
@ -150,10 +153,11 @@ function FileCard({file}) {
<CardContent {...s}>
<CardHeader>
<ExternalCardLink to={downloadUrl}>
{cardTitleWrapper(file.title || file.stem || file.path)}
{cardTitleWrapper(file.title || file.stem || file.primary_path)}
</ExternalCardLink>
</CardHeader>
<p>{isoDatetimeToString(file.modified / 1000)}</p>
{author && <b {...s}>{author}</b>}
<p>{isoDatetimeToString(file.modified)}</p>
<p>{size}</p>
</CardContent>
</Card>
@ -162,7 +166,7 @@ function FileCard({file}) {
export function FileCards({files}) {
if (!_.isEmpty(files)) {
return <CardGroupCentered>
{files.map(i => <FileCard key={i['path']} file={i}/>)}
{files.map(i => <FileCard key={i['primary_path']} file={i}/>)}
</CardGroupCentered>
} else if (files && files.length === 0) {
return <Segment>No results!</Segment>
@ -172,7 +176,7 @@ export function FileCards({files}) {
}
function ImageRowCells({file}) {
const url = `/media/${encodeURIComponent(file.path)}`;
const url = `/media/${encodeURIComponent(file.primary_path)}`;
let poster = <FileIcon file={file} size='large'/>;
if (file.size && file.size < 50000000) {
@ -189,18 +193,18 @@ function ImageRowCells({file}) {
</TableCell>
<TableCell>
<PreviewLink file={file}>
<p>{textEllipsis(file.title || file.stem || file.path)}</p>
<p>{textEllipsis(file.title || file.stem || file.primary_path)}</p>
</PreviewLink>
</TableCell>
</React.Fragment>
}
export function EbookRowCells({file}) {
const {ebook} = file;
const {data} = file;
let cover = <CardIcon><FileIcon file={file}/></CardIcon>;
if (ebook && ebook.cover_path) {
const coverSrc = `/media/${encodeURIComponent(ebook.cover_path)}`;
if (data && data.cover_path) {
const coverSrc = `/media/${encodeURIComponent(data.cover_path)}`;
cover = <Image wrapped src={coverSrc} width='50px'/>;
}
@ -211,7 +215,7 @@ export function EbookRowCells({file}) {
</TableCell>
<TableCell>
<PreviewLink file={file}>
{textEllipsis(ebook.title || file.title)}
{textEllipsis(file.title || file.stem)}
</PreviewLink>
</TableCell>
</React.Fragment>
@ -224,12 +228,12 @@ function FileRow({file}) {
if (file.model === 'video' && 'video' in file) {
return <VideoRowCells file={file}/>;
} else if (file.model === 'archive' && 'archive' in file) {
} else if (file.model === 'archive') {
return <ArchiveRowCells file={file}/>;
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
return <ImageRowCells file={file}/>;
} else if (isEbookType) {
return <EbookRowCells key={file['path']} file={file}/>;
return <EbookRowCells key={file['primary_path']} file={file}/>;
}
// Fragment for SelectableRow
@ -266,6 +270,14 @@ export function FileTable({files, selectOn, onSelect, footer, selectedKeys}) {
}
}
export function FileRowTagIcon({file}) {
if (file.tags && file.tags.length) {
return <Label circular color='green' style={{padding: '0.5em', marginRight: '0.5em'}}>
<Icon name='tag' style={{margin: 0}}/>
</Label>;
}
}
export function FilesView({
files,
limit,
@ -347,19 +359,20 @@ export function FilesView({
const filtersDropdown = <Dropdown selection clearable search
multiple={multipleFilters || undefined}
options={filterOptions}
placeholder={filterPlaceholder || 'Filters'}
options={filterOptions || []}
placeholder={filterPlaceholder || 'Files Types'}
onChange={(e, {value}) => setFilters(value)}
style={{marginLeft: '0.3em'}}
style={{marginLeft: '0.3em', marginTop: '0.3em'}}
value={activeFilters}
/>;
return <>
<Grid columns={menuColumnsCount || 1} stackable>
<Grid.Column mobile={8} computer={6}>
<Grid.Column mobile={16} computer={12}>
{selectButton}
{viewButton}
{limitDropdown}
<TagsFilesGroupDropdown/>
{filterOptions && filtersDropdown}
</Grid.Column>
{menuColumns}
@ -369,6 +382,38 @@ export function FilesView({
</>
}
export function TagsFilesGroupDropdown({onChange}) {
// Creates a dropdown that the User can use to manipulate the tag query.
const {searchParams, updateQuery} = useQuery();
const activeTag = searchParams.get('tag');
const {tags} = React.useContext(TagsContext);
const localOnChange = (name) => {
updateQuery({'tag': name});
if (onChange) {
onChange(name);
}
}
let tagOptions = [];
if (tags && tags.length) {
for (let i = 0; i < tags.length; i++) {
const {name} = tags[i];
tagOptions = [...tagOptions, {key: name, text: name, value: name}];
}
return <Dropdown selection clearable search
options={tagOptions || []}
placeholder={'Tags'}
onChange={(e, {value}) => localOnChange(value)}
style={{marginLeft: '0.3em', marginTop: '0.3em'}}
value={activeTag}
/>;
} else {
return <React.Fragment/>
}
}
export function FilesSearchView({
showView = true,
showLimit = false,
@ -391,7 +436,6 @@ export function FilesSearchView({
const {searchFiles, limit, setLimit, totalPages, activePage, setPage} =
useSearchFiles(24, emptySearch, model);
const {searchParams, updateQuery} = useQuery();
const setView = (value) => updateQuery({view: value});
const view = searchParams.get('view');

View file

@ -19,6 +19,7 @@ import {useChannel} from "../hooks/customHooks";
import {ThemeContext} from "../contexts/contexts";
import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme";
import {VideoCard} from "./Videos";
import {TagsDisplay} from "../Tags";
const MEDIA_PATH = '/media';
@ -26,6 +27,7 @@ const MEDIA_PATH = '/media';
function videoFileLink(path, directory = false) {
if (path) {
const href = directory ?
// Add / to end of directory.
`${MEDIA_PATH}/${encodeURIComponent(path)}/` :
`${MEDIA_PATH}/${encodeURIComponent(path)}`;
return <a href={href} target='_blank' rel='noopener noreferrer'>
@ -36,7 +38,7 @@ function videoFileLink(path, directory = false) {
}
}
function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
const {theme} = useContext(ThemeContext);
const navigate = useNavigate();
@ -84,30 +86,12 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
let vttCaptions = video.caption_path && video.caption_path.endsWith('.vtt');
let description = 'No description available.';
let viewCount = video.view_count;
if (video.info_json && video.info_json['description']) {
if (video && video['description']) {
// Only replace empty description if there is one available.
description = video.info_json['description'];
description = video['description'];
description = chaptersInDescription(description, setVideoTime)
viewCount = viewCount || video.info_json['view_count'];
}
const favorite = video && video.favorite;
let handleFavorite = async (e) => {
e.preventDefault();
if (video) {
setFavorite(!favorite);
}
}
const favorite_button = (<Button
color='green'
style={{'margin': '0.5em'}}
onClick={handleFavorite}
>
<Icon name='heart'/>
{favorite ? 'Unfavorite' : 'Favorite'}
</Button>);
let descriptionPane = {
menuItem: 'Description', render: () => <TabPane>
<pre className="wrap-text">
@ -119,13 +103,13 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
let aboutPane = {
menuItem: 'About', render: () => <TabPane>
<h3>Size</h3>
<p>{video.size ? humanFileSize(video.size) : 'Unknown'}</p>
<p>{videoFile.size ? humanFileSize(videoFile.size) : 'Unknown'}</p>
<h3>Source URL</h3>
<p>{video.url ? <a href={video.url}>{video.url}</a> : 'N/A'}</p>
<h3>View Count</h3>
<p>{viewCount ? humanNumber(viewCount) : 'N/A'}</p>
<p>{video.view_count ? humanNumber(video.view_count) : 'N/A'}</p>
<h3>Censored</h3>
<p>{video.censored ? 'Yes' : 'No'}</p>
@ -138,25 +122,30 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
</TabPane>
};
const {poster_file, caption_files, info_json_file} = video;
let filesPane = {
menuItem: 'Files', render: () => <TabPane>
<h3>Video File</h3>
{videoFileLink(video['video_path'])}
<h4>Info JSON File</h4>
<PreviewPath
path={video['info_json_path']}
mimetype='application/json'>{video['info_json_path']}</PreviewPath>
{info_json_file &&
<PreviewPath
path={info_json_file['path']}
mimetype={info_json_file['mimetype']}>{info_json_file['path']}</PreviewPath>
}
<h4>Caption File</h4>
<PreviewPath
path={video['caption_path']}
mimetype='text/'>{video['caption_path']}</PreviewPath>
<h4>Caption Files</h4>
{caption_files &&
caption_files.map(i => <PreviewPath key={i['path']} {...i}>{i['path']}</PreviewPath>)
}
<h4>Poster File</h4>
<PreviewPath
path={video['poster_path']}
mimetype='image/'>{video['poster_path']}</PreviewPath>
{poster_file &&
<PreviewPath
path={poster_file['path']}
mimetype={poster_file['mimetype']}>{poster_file['path']}</PreviewPath>
}
<h4>Directory</h4>
{videoFileLink(videoFile['directory'], true)}
@ -188,7 +177,7 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
<Container style={{marginTop: '1em'}}>
<Segment>
<Header as='h2'>{video.title || video.video_path}</Header>
<Header as='h2'>{videoFile.title || videoFile.stem || video.video_path}</Header>
{video.upload_date && <h3>{isoDatetimeToString(video.upload_date)}</h3>}
<h3>
{channel && <Link to={`/videos/channel/${channel.id}/video`}>
@ -197,7 +186,6 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
</h3>
<p>
{favorite_button}
<a href={downloadUrl} target='_blank' rel='noopener noreferrer'>
<Button style={{margin: '0.5em'}}>
<Icon name='download'/>
@ -217,6 +205,11 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
onConfirm={() => handleDeleteVideo(video.id)}
/>
</p>
<br/>
</Segment>
<Segment>
<TagsDisplay fileGroup={videoFile} onClick={fetchVideo}/>
</Segment>
<Tab menu={tabMenu} panes={tabPanes}/>

View file

@ -34,26 +34,27 @@ import {
} from "semantic-ui-react";
import {Channels, EditChannel, NewChannel} from "./Channels";
import {useChannel, useQuery, useSearchVideos, useVideo, useVideoStatistics} from "../hooks/customHooks";
import {FilesView} from "./Files";
import {FileRowTagIcon, FilesView} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {Button, Card, CardIcon, Header, Loader, Placeholder, Segment, Statistic, StatisticGroup} from "./Theme";
import {deleteVideos} from "../api";
import {ThemeContext} from "../contexts/contexts";
import _ from "lodash";
import {taggedImageLabel} from "../Tags";
export function VideoWrapper() {
const {videoId} = useParams();
const {videoFile, prevFile, nextFile, setFavorite} = useVideo(videoId);
const {videoFile, prevFile, nextFile, fetchVideo} = useVideo(videoId);
// Scroll to the top when videoId changes.
useEffect(scrollToTop, [videoId]);
return <VideoPage videoFile={videoFile} prevFile={prevFile} nextFile={nextFile}
setFavorite={setFavorite} autoplay={true}/>
return <VideoPage videoFile={videoFile} prevFile={prevFile} nextFile={nextFile} fetchVideo={fetchVideo}
autoplay={true}/>
}
function VideosPage({filter}) {
function VideosPage() {
const {channelId} = useParams();
const {searchParams, updateQuery} = useQuery();
@ -69,31 +70,19 @@ function VideosPage({filter}) {
searchOrder = defaultSearchOrder;
}
let filtersEnabled = searchParams.getAll('filter');
filtersEnabled = filter ? [...filtersEnabled, filter] : filtersEnabled;
const {searchStr, setSearchStr, videos, activePage, setPage, limit, setLimit, totalPages, setOrderBy, fetchVideos} =
useSearchVideos(null, channelId, searchOrder, filtersEnabled);
useSearchVideos(null, channelId, searchOrder);
const setView = (value) => updateQuery({view: value});
const view = searchParams.get('view');
const {channel} = useChannel(channelId);
let title = 'Videos';
if (filter && filter.indexOf('favorite') >= 0) {
title = 'Favorite Videos';
}
if (channel && channel.name) {
title = `${channel.name} Videos`;
}
useTitle(title);
const filterOptions = [
{text: 'Favorites', key: 'favorite', value: 'favorite'},
{text: 'Censored', key: 'censored', value: 'censored'},
];
const setFilters = (value) => updateQuery({filter: value});
let videoOrders = [
{key: '-upload_date', value: '-upload_date', text: 'Newest'},
{key: 'upload_date', value: 'upload_date', text: 'Oldest'},
@ -219,10 +208,6 @@ function VideosPage({filter}) {
setView={setView}
setLimit={setLimit}
setPage={setPage}
filterOptions={filterOptions}
activeFilters={filtersEnabled}
setFilters={setFilters}
multipleFilters={true}
menuColumnsCount={3}
menuColumns={menuColumns}
>
@ -242,7 +227,6 @@ function Statistics() {
const videoNames = [
{key: 'videos', label: 'Videos'},
{key: 'favorites', label: 'Favorite Videos'},
{key: 'sum_size', label: 'Total Size'},
{key: 'max_size', label: 'Largest Video'},
{key: 'week', label: 'Downloads Past Week'},
@ -283,7 +267,6 @@ function Statistics() {
export function VideosRoute(props) {
const links = [
{text: 'Videos', to: '/videos', key: 'videos', end: true},
{text: 'Favorites', to: '/videos/favorites', key: 'favorites'},
{text: 'Channels', to: '/videos/channel', key: 'channel'},
{text: 'Statistics', to: '/videos/statistics', key: 'statistics'},
];
@ -292,7 +275,6 @@ export function VideosRoute(props) {
<TabLinks links={links}/>
<Routes>
<Route path='/' exact element={<VideosPage/>}/>
<Route path='favorites' exact element={<VideosPage filter='favorite' header='Favorite Videos'/>}/>
<Route path='channel' exact element={<Channels/>}/>
<Route path='statistics' exact element={<Statistics/>}/>
<Route path='channel/new' exact element={<NewChannel/>}/>
@ -303,7 +285,7 @@ export function VideosRoute(props) {
}
export function VideoCard({file}) {
const {video} = file;
const {video, tags} = file;
const {s} = useContext(ThemeContext);
let video_url = `/videos/video/${video.id}`;
@ -317,10 +299,7 @@ export function VideoCard({file}) {
}
const poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;
let imageLabel = null;
if (video.favorite) {
imageLabel = {corner: 'left', icon: 'heart', color: 'green'};
}
let imageLabel = !_.isEmpty(tags) ? taggedImageLabel : null;
let poster = <Link to={video_url}><CardIcon><FileIcon file={file}/></CardIcon></Link>;
if (poster_url) {
poster = <CardPosterLink to={video_url} poster_url={poster_url} imageLabel={imageLabel}/>;
@ -334,7 +313,7 @@ export function VideoCard({file}) {
<CardHeader>
<Container textAlign='left'>
<Link to={video_url} className="no-link-underscore card-link">
<p {...s}>{cardTitleWrapper(video.title || video.stem || video.video_path)}</p>
<p {...s}>{cardTitleWrapper(file.title || file.name || video.stem || video.video_path)}</p>
</Link>
</Container>
</CardHeader>
@ -357,12 +336,10 @@ export function VideoRowCells({file}) {
const poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;
let poster;
let imageLabel = video.favorite ? {corner: 'left', icon: 'heart', color: 'green'} : null;
if (poster_url) {
poster = <CardLink to={video_url}>
<Image wrapped
src={poster_url}
label={imageLabel}
width='50px'
/>
</CardLink>
@ -379,6 +356,7 @@ export function VideoRowCells({file}) {
</TableCell>
<TableCell>
<CardLink to={video_url}>
<FileRowTagIcon file={file}/>
{textEllipsis(video.title || video.stem || video.video_path)}
</CardLink>
</TableCell>

View file

@ -51,7 +51,7 @@ class WROLMode extends React.Component {
want to prevent any potential loss of data.
</Header>
<p {...t}>
Note: User settings and favorites can still be modified.
Note: User settings and tags can still be modified.
</p>
<Toggle
checked={this.state.WROLMode}

View file

@ -1,6 +1,5 @@
import {useContext, useEffect, useRef, useState} from "react";
import {
favoriteVideo,
fetchDomains,
filesSearch,
getArchive,
@ -81,17 +80,17 @@ export const useDomains = () => {
}
export const useArchive = (archiveId) => {
const [archiveFile, setArchiveFile] = useState(null);
const [archiveFileGroup, setArchiveFileGroup] = useState(null);
const [history, setHistory] = useState(null);
const fetchArchive = async () => {
try {
const [file, history] = await getArchive(archiveId);
setArchiveFile(file);
const [file_group, history] = await getArchive(archiveId);
setArchiveFileGroup(file_group);
setHistory(history);
} catch (e) {
console.error(e);
setArchiveFile(undefined);
setArchiveFileGroup(undefined);
}
}
@ -99,7 +98,7 @@ export const useArchive = (archiveId) => {
fetchArchive();
}, [archiveId]);
return {archiveFile, history};
return {archiveFile: archiveFileGroup, history, fetchArchive};
}
export const useQuery = () => {
@ -152,6 +151,7 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
const {searchParams, updateQuery} = useQuery();
const searchStr = searchParams.get('q') || '';
const order = searchParams.get('order') || order_by;
const activeTags = searchParams.get('tag');
const [archives, setArchives] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@ -160,7 +160,7 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
setArchives(null);
setTotalPages(0);
try {
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order);
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags);
setArchives(archives_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -177,7 +177,7 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
useEffect(() => {
localSearchArchives();
}, [searchStr, limit, domain, order, activePage]);
}, [searchStr, limit, domain, order, activePage, activeTags]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -204,11 +204,12 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
}
}
export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
export const useSearchVideos = (defaultLimit, channelId, order_by) => {
const {searchParams, updateQuery} = useQuery();
const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit)
const searchStr = searchParams.get('q') || '';
const order = searchParams.get('order') || order_by;
const activeTags = searchParams.get('tag');
const [videos, setVideos] = useState();
const [totalPages, setTotalPages] = useState(0);
@ -217,7 +218,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
setVideos(null);
setTotalPages(0);
try {
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, filters);
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags);
setVideos(videos_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -234,7 +235,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
useEffect(() => {
localSearchVideos();
}, [searchStr, limit, channelId, offset, order_by, filters.join('')]);
}, [searchStr, limit, channelId, offset, order_by, activeTags]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -258,6 +259,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
setSearchStr,
setOrderBy,
fetchVideos: localSearchVideos,
activeTags,
}
}
@ -292,24 +294,11 @@ export const useVideo = (videoId) => {
}
}
const setFavorite = async (value) => {
if (!videoFile) {
console.error('No video to favorite');
return
}
try {
await favoriteVideo(videoFile.video.id, value);
fetchVideo();
} catch (e) {
console.error(e);
}
}
useEffect(() => {
fetchVideo();
}, [videoId]);
return {videoFile, prevFile, nextFile, setFavorite};
return {videoFile, prevFile, nextFile, fetchVideo};
}
export const useChannel = (channel_id) => {
@ -394,22 +383,23 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
const searchStr = searchParams.get('q');
const filter = searchParams.get('filter');
const model_ = searchParams.get('model');
const activeTags = searchParams.get('tag');
const [searchFiles, setSearchFiles] = useState(null);
const [totalPages, setTotalPages] = useState(0);
const [activePage, setActivePage] = useState(calculatePage(offset, limit));
const localSearchFiles = async () => {
if (!emptySearch && !searchStr) {
if (!emptySearch && !searchStr && !activeTags) {
return;
}
const mimetypes = filterToMimetypes(filter);
setSearchFiles(null);
setTotalPages(0);
try {
let [files, total] = await filesSearch(
offset, limit, searchStr, mimetypes, model || model_);
setSearchFiles(files);
let [file_groups, total] = await filesSearch(
offset, limit, searchStr, mimetypes, model || model_, activeTags);
setSearchFiles(file_groups);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
console.error(e);
@ -424,7 +414,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
useEffect(() => {
localSearchFiles();
}, [searchStr, limit, offset, activePage, filter, model, model_]);
}, [searchStr, limit, offset, activePage, filter, model, model_, activeTags]);
const setPage = (i) => {
i = parseInt(i);
@ -442,7 +432,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
updateQuery({l: value, o: 0});
}
return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit};
return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit, activeTags};
}
export const useBrowseFiles = () => {

View file

@ -33,16 +33,16 @@ services:
# healthcheck:
# test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]
archive:
build:
context: .
dockerfile: docker/archive/Dockerfile
volumes:
- './docker/archive/main.py:/app/main.py'
ports:
- '8083:8080'
healthcheck:
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
# archive:
# build:
# context: .
# dockerfile: docker/archive/Dockerfile
# volumes:
# - './docker/archive/main.py:/app/main.py'
# ports:
# - '8083:8080'
# healthcheck:
# test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
app:
depends_on:

32
main.py
View file

@ -10,7 +10,7 @@ from sanic.signals import Event
from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS
from wrolpi import root_api, admin
from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, \
wrol_mode_enabled, cancel_refresh_tasks
wrol_mode_enabled, cancel_refresh_tasks, set_log_level
from wrolpi.downloader import download_manager, import_downloads_config
from wrolpi.root_api import api_app
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
@ -115,8 +115,14 @@ def main():
return 1
logger.warning(f'Starting with: {sys.argv}')
set_log_level(args)
logger.debug(get_version_string())
if args.verbose == 1:
set_log_level(logging.INFO)
elif args.verbose and args.verbose == 2:
set_log_level(logging.DEBUG)
elif args.verbose and args.verbose >= 3:
# Log everything. Add SQLAlchemy debug logging.
set_log_level(logging.NOTSET)
logger.info(get_version_string())
if DOCKERIZED:
logger.info('Running in Docker')
@ -151,26 +157,6 @@ def main():
return root_api.main(args)
def set_log_level(args):
"""
Set the level at the root logger so all children that have been created (or will be created) share the same level.
"""
root_logger = logging.getLogger()
sa_logger = logging.getLogger('sqlalchemy.engine')
if args.verbose == 1:
root_logger.setLevel(logging.INFO)
elif args.verbose and args.verbose == 2:
root_logger.setLevel(logging.DEBUG)
elif args.verbose and args.verbose >= 3:
root_logger.setLevel(logging.DEBUG)
sa_logger.setLevel(logging.DEBUG)
# Always warn about the log level so we know what will be logged
effective_level = logger.getEffectiveLevel()
level_name = logging.getLevelName(effective_level)
logger.warning(f'Logging level: {level_name}')
@api_app.before_server_start
@limit_concurrent(1)
async def startup(app: Sanic):

View file

@ -1,20 +1,20 @@
import asyncio
import json
import pathlib
from abc import ABC
from typing import Tuple, Dict, List
from typing import Tuple, List
from sqlalchemy import not_
from sqlalchemy.orm import Session
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
from wrolpi.db import optional_session, get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.indexers import register_indexer, Indexer
from wrolpi.files.models import File
from wrolpi.errors import UnrecoverableDownloadError, InvalidArchive
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
from . import lib
from .api import bp # noqa
from .lib import match_archive_files
from .lib import is_singlefile_file, get_title_from_html
from .models import Archive, Domain
PRETTY_NAME = 'Archive'
@ -55,58 +55,129 @@ class ArchiveDownloader(Downloader, ABC):
archive_downloader = ArchiveDownloader(priority=100)
def find_archive_file_in_group(group: List[File]):
for file in group:
if lib.is_singlefile_file(file.path):
return file
def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
file_group_id = file_group.id
if not file_group_id:
session.flush([file_group])
file_group_id = file_group.id
# All Archives have an HTML Singlefile.
html_paths = file_group.my_paths('text/html')
if not html_paths:
logger.error('Query returned a group without an HTML file!')
raise InvalidArchive('FileGroup does not contain any html files')
# All Archives have a Singlefile.
for file in html_paths:
if is_singlefile_file(file):
singlefile_path = file
break
else:
logger.debug(f'No Archive singlefile found in {file_group}')
raise InvalidArchive('FileGroup does not contain a singlefile')
readability_json_path = None
json_files = file_group.my_json_files()
for json_file in json_files:
path = json_file['path']
if path.name.endswith('.readability.json'):
readability_json_path = path
break
readability_txt_path = None
text_files = file_group.my_text_files()
for text_file in text_files:
path = text_file['path']
if path.name.endswith('.readability.txt'):
readability_txt_path = path
break
try:
file_group.model = 'archive'
file_group.primary_path = singlefile_path
title = None
if readability_json_path:
title = get_title(readability_json_path)
if not title:
title = get_title_from_html(singlefile_path.read_text())
contents = None
if readability_txt_path:
contents = get_article(readability_txt_path)
archive = Archive(file_group_id=file_group_id, file_group=file_group)
session.add(archive)
archive.validate()
session.flush([archive])
file_group.title = file_group.a_text = title
file_group.d_text = contents
file_group.data = {
'id': archive.id,
'archive_datetime': archive.archive_datetime,
'domain': archive.domain.domain if archive.domain else None,
'readability_json_path': archive.readability_json_path,
'readability_path': archive.readability_path,
'readability_txt_path': archive.readability_txt_path,
'screenshot_path': archive.screenshot_path,
'singlefile_path': archive.singlefile_path,
'url': archive.url,
}
return archive
except Exception as e:
logger.error(f'Failed to model Archive {file_group}', exc_info=e)
if PYTEST:
raise
@register_modeler
def archive_modeler(groups: Dict[str, List[File]], session: Session):
archive_files = {stem: file for stem, group in groups.items() if (file := find_archive_file_in_group(group))}
if not archive_files:
# No archives in these groups.
return
# Get all matching Archive records (if any) in one query.
archive_paths = [i.path for i in archive_files.values()]
archive_records = {i.singlefile_path: i for i in
session.query(Archive).filter(Archive.singlefile_path.in_(archive_paths))}
async def archive_modeler():
invalid_archives = set()
for stem, archive_file in archive_files.items():
group = groups[stem]
while True:
with get_db_session(commit=True) as session:
results = session.query(FileGroup, Archive) \
.filter(
# Get all groups that contain a PDF that have not been indexed.
FileGroup.indexed != True,
FileGroup.mimetype == 'text/html',
).filter(not_(FileGroup.id.in_(list(invalid_archives)))) \
.outerjoin(Archive, Archive.file_group_id == FileGroup.id) \
.limit(20)
readability_file, singlefile_file, readability_json_file, readability_txt_file, screenshot_file = \
match_archive_files(group)
processed = 0
for file_group, archive in results:
processed += 1
archive: Archive = archive_records.get(archive_file.path)
if not archive:
archive = Archive(singlefile_file=singlefile_file)
session.add(archive)
archive.readability_file = readability_file
archive.readability_json_file = readability_json_file
archive.readability_txt_file = readability_txt_file
archive.screenshot_file = screenshot_file
if archive:
try:
archive_id = archive.id
archive.validate()
except Exception:
# Archive could not be indexed.
invalid_archives |= {file_group.id, }
if archive.readability_file:
archive.readability_file.associated = True
archive.readability_file.do_stats()
if archive.readability_json_file:
archive.readability_json_file.associated = True
archive.readability_json_file.do_stats()
if archive.readability_txt_file:
archive.readability_txt_file.associated = True
archive.readability_txt_file.do_stats()
if archive.screenshot_file:
archive.screenshot_file.associated = True
archive.screenshot_file.do_stats()
logger.error(f'Unable to validate Archive {archive_id}')
if PYTEST:
raise
else:
try:
model_archive(file_group, session=session)
except InvalidArchive:
# FileGroup was not an archive group.
invalid_archives |= {file_group.id, }
archive.singlefile_file.model = Archive.__tablename__
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
file_group.indexed = True
archive.singlefile_file.do_index()
archive.validate()
if processed < 20:
# Did not reach limit, do not query again.
break
# Remove this group, it will not be processed by other modelers.
del groups[stem]
# Sleep to catch cancel.
await asyncio.sleep(0)
@register_after_refresh
@ -124,7 +195,13 @@ def archive_cleanup():
deleted = 0
for archive in archives:
processed += 1
if not lib.is_singlefile_file(archive.singlefile_path):
singlefile_path = archive.singlefile_path
if not singlefile_path:
archive.file_group.model = None
session.delete(archive)
deleted += 1
continue
if not lib.is_singlefile_file(singlefile_path):
# Archive is not a real archive.
archive.singlefile_file.model = None
session.delete(archive)
@ -142,45 +219,18 @@ def archive_cleanup():
session.delete(domain)
@register_indexer('text/html')
class ArchiveIndexer(Indexer, ABC):
"""Gathers index data from the files associated with an Archive."""
def get_title(path):
with path.open('rt') as fh:
try:
return json.load(fh)['title']
except Exception:
if not PYTEST:
logger.warning(f'Archive readability json file exists, but cannot get title. {path}')
return None
@staticmethod
def get_title(file: File):
singlefile_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
readability_json_path = singlefile_path.with_suffix('.readability.json')
if readability_json_path.is_file():
with readability_json_path.open('rt') as fh:
try:
return json.load(fh)['title']
except Exception:
if not PYTEST:
logger.warning(f'Archive readability json file exists, but cannot get title. {file}')
return None
@staticmethod
def get_article(file: File):
singlefile_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
readability_txt_path = singlefile_path.with_suffix('.readability.txt')
if readability_txt_path.is_file():
return readability_txt_path.read_text()
@classmethod
def create_index(cls, file: File) -> Tuple:
"""
Index an Archive file and it's associated files. This mimetype matches non-archive (non-singlefile) html files,
so this index will return empty if a readability file is found.
a = title
b = <empty>
c = <empty>
d = readability article
"""
if file.path.name.endswith('.readability.html'):
# Readability files are not the primary Archive file.
return None, None, None, None
a = cls.get_title(file)
d = cls.get_article(file)
d = split_lines_by_length(d)
return a, None, None, d
def get_article(path):
if path.is_file():
text = path.read_text()
text = split_lines_by_length(text)
return text

View file

@ -22,9 +22,9 @@ logger = logger.getChild(__name__)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
async def get_archive(_: Request, archive_id: int):
archive = lib.get_archive(archive_id=archive_id)
archive_file = archive.singlefile_file.__json__()
history = [i.singlefile_file.__json__() for i in archive.history]
return json_response({'file': archive_file, 'history': history})
archive_file_group = archive.file_group.__json__()
history = [i.file_group.__json__() for i in archive.history]
return json_response({'file_group': archive_file_group, 'history': history})
@bp.delete('/<archive_ids:int>')
@ -52,7 +52,7 @@ archive_limit_limiter = api_param_limiter(100)
@bp.post('/search')
@openapi.definition(
summary='Search archive contents and titles',
summary='A File search with more filtering related to Archives',
body=schema.ArchiveSearchRequest,
)
@openapi.response(HTTPStatus.OK, schema.ArchiveSearchResponse)
@ -64,6 +64,6 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
limit = archive_limit_limiter(body.limit)
offset = body.offset or 0
archives, total = lib.archive_search(search_str, domain, limit, offset, body.order_by)
ret = dict(archives=archives, totals=dict(archives=total))
file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)

View file

@ -1,4 +1,3 @@
import copy
import datetime
import json
import pathlib
@ -9,7 +8,6 @@ import pytz
from modules.archive.lib import archive_strftime
from modules.archive.models import Archive, Domain
from wrolpi.files.models import File
@pytest.fixture
@ -68,45 +66,17 @@ def archive_factory(test_session, archive_directory, make_files_structure):
test_session.add(domain)
test_session.flush([domain])
archive = Archive(
readability_json_file=File(path=readability_json_path) if readability_json_path else None,
readability_file=File(path=readability_path) if readability_path else None,
readability_txt_file=File(path=readability_txt_path) if readability_txt_path else None,
screenshot_file=File(path=screenshot_path) if screenshot_path else None,
singlefile_file=File(path=singlefile_path, model='archive'),
title=title,
url=url,
domain_id=domain.id if domain else None,
archive_datetime=archive_datetime,
)
test_session.add(archive)
archive.singlefile_file.do_index()
# Only add files that were created.
files = (readability_path, readability_json_path, readability_txt_path, screenshot_path, singlefile_path)
files = list(filter(None, files))
archive = Archive.from_paths(test_session, *files)
archive.url = url
archive.title = title
archive.archive_datetime = next(now)
archive.domain = domain
archive.validate()
return archive
return _
SINGLEFILE_CONTENTS = '''<html><!--
Page saved with SingleFile
url: https://example.com
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">
<meta name="msapplication-TileColor" content="#2d89ef">
<meta name="theme-color" content="#000000">
<title>the title</title>
<body>
some body contents
</body>
</html>
'''
@pytest.fixture
def singlefile_contents() -> str:
"""Return a short HTML string that contains an example Singlefile file."""
return copy.deepcopy(SINGLEFILE_CONTENTS)

View file

@ -7,22 +7,19 @@ import subprocess
import tempfile
from dataclasses import dataclass
from datetime import datetime
from functools import partial
from itertools import groupby
from typing import Iterator, Optional, Tuple, List
from typing import Optional, Tuple, List
from bs4 import BeautifulSoup
from selenium import webdriver
from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, walk, \
aiohttp_post, match_paths_to_suffixes, iterify
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy
from wrolpi.files.lib import handle_search_results
from wrolpi.files.models import File
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
from wrolpi.vars import DOCKERIZED, PYTEST
logger = logger.getChild(__name__)
@ -52,8 +49,6 @@ class ArchiveFiles:
def get_archive_directory() -> pathlib.Path:
archive_directory = get_media_directory() / 'archive'
if not archive_directory.is_dir():
archive_directory.mkdir()
return archive_directory
@ -66,7 +61,7 @@ def get_domain_directory(url: str) -> pathlib.Path:
elif directory.exists():
raise FileNotFoundError(f'Domain directory {directory} is already a file')
directory.mkdir()
directory.mkdir(parents=True, exist_ok=True)
return directory
@ -265,45 +260,23 @@ async def do_archive(url: str) -> Archive:
fh.write(json.dumps(readability))
# Create any File models that we can, index them all.
singlefile_file = File(path=archive_files.singlefile, model='archive')
singlefile_file.do_index()
readability_file = None
if archive_files.readability:
readability_file = File(path=archive_files.readability, associated=True)
readability_file.do_index()
readability_json_file = None
if archive_files.readability_json:
readability_json_file = File(path=archive_files.readability_json, associated=True)
readability_json_file.do_index()
readability_txt_file = None
if archive_files.readability_txt:
readability_txt_file = File(path=archive_files.readability_txt, associated=True)
readability_txt_file.do_index()
screenshot_file = None
if archive_files.screenshot:
screenshot_file = File(path=archive_files.screenshot, associated=True)
screenshot_file.do_index()
paths = (
archive_files.singlefile, archive_files.readability, archive_files.readability_json,
archive_files.readability_txt,
archive_files.screenshot)
paths = list(filter(None, paths))
with get_db_session(commit=True) as session:
domain = get_or_create_domain(session, url)
archive = Archive(
title=title,
archive_datetime=now(),
singlefile_file=singlefile_file,
readability_file=readability_file,
readability_json_file=readability_json_file,
readability_txt_file=readability_txt_file,
screenshot_file=screenshot_file,
url=url,
domain_id=domain.id,
)
session.add(archive)
archive = Archive.from_paths(session, *paths)
archive.archive_datetime = now()
archive.url = url
archive.domain = get_or_create_domain(session, url)
session.flush()
return archive
def get_or_create_domain(session, url) -> Domain:
def get_or_create_domain(session: Session, url) -> Domain:
"""
Get/create the Domain for this archive.
"""
@ -348,132 +321,11 @@ def delete_archives(*archive_ids: List[int]):
archive.delete()
def archive_strptime(dt: str) -> datetime:
try:
return datetime.strptime(dt, '%Y-%m-%d-%H-%M-%S')
except ValueError:
return datetime.strptime(dt, '%Y-%m-%d %H:%M:%S')
def archive_strftime(dt: datetime) -> str:
return dt.strftime('%Y-%m-%d-%H-%M-%S')
def group_archive_files(files: Iterator[pathlib.Path]) -> groupby:
"""
Group archive files by their timestamp.
"""
# groupby requires the files to be sorted.
files = sorted(files)
# Group archive files by their datetime at the beginning of the file.
groups = groupby(files, key=lambda i: i.name[:19])
for dt, files in groups:
try:
dt = archive_strptime(dt)
except ValueError:
logger.info(f'Ignoring invalid archives of {dt=}')
continue
# Sort the files into their respective slots.
archive_files = ArchiveFiles()
file = None
for file in files:
name = file.name
# TODO remove the -readability matching once migrated.
if name.endswith('.readability.html') or name.endswith('-readability.html'):
archive_files.readability = file
elif name.endswith('.html'):
archive_files.singlefile = file
elif name.endswith('.png') or name.endswith('.jpg') or name.endswith('.jpeg'):
archive_files.screenshot = file
elif name.endswith('.readability.json') or name.endswith('-readability.json'):
archive_files.readability_json = file
elif name.endswith('.readability.txt') or name.endswith('-readability.txt'):
archive_files.readability_txt = file
if not archive_files.singlefile:
logger.warning(f'Archive does not have a singlefile html! Ignoring. {file}')
continue
if not archive_files.readability_json:
logger.warning(f'Archive does not have a json file! Ignoring. {file}')
continue
yield dt, archive_files
OLD_ARCHIVE_MATCHER = re.compile(r'\d{4}-\d\d-\d\d (\d\d:){2}\d\d\.\d{6}.*$')
def migrate_archive_files():
"""
Rename Archive files from "YYYY-mm-dd HH:MM:SS.ZZZZZ.html" to "YYYY-mm-dd-HH-MM-SS_{TITLE}.html" and all their
associated files.
"""
archive_directory = get_archive_directory()
def _is_archive_file(path: pathlib.Path) -> bool:
return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(OLD_ARCHIVE_MATCHER.match(path.name))
plan = []
# It is safer to plan the renames before we make them.
for domain_directory in filter(lambda i: i.is_dir(), archive_directory.iterdir()):
all_archives_files = filter(_is_archive_file, walk(domain_directory))
archive_groups = group_archive_files(all_archives_files)
for dt, archive_files in archive_groups:
archive_files: ArchiveFiles
with archive_files.readability_json.open() as fh:
title = escape_file_name(json.load(fh).get('title') or 'NA')
title = title[:50]
dt = archive_strftime(dt)
prefix = f'{dt}_{title}'
singlefile_path = domain_directory / f'{prefix}.html'
readability_path = domain_directory / f'{prefix}.readability.html'
readability_txt_path = domain_directory / f'{prefix}.readability.txt'
readability_json_path = domain_directory / f'{prefix}.readability.json'
screenshot_path = domain_directory / f'{prefix}.png'
# Every Archive is required to have these files.
plan.append((archive_files.singlefile, singlefile_path))
plan.append((archive_files.readability_json, readability_json_path))
# These files are optional.
if archive_files.readability:
plan.append((archive_files.readability, readability_path))
if archive_files.readability_txt:
plan.append((archive_files.readability_txt, readability_txt_path))
if archive_files.screenshot:
plan.append((archive_files.screenshot, screenshot_path))
if not plan:
logger.info('Could not find any Archive files to migrate. Was it already performed?')
return plan
# Check that all new files do not exist.
for old, new in plan:
if new.exists():
raise FileExistsError(f'Cannot migrate archive files! {new} already exists!')
# Finally, move all the files now that its safe.
for old, new in plan:
old.rename(new)
return plan
ARCHIVE_MATCHER = re.compile(r'\d{4}(-\d\d){5}_.*$')
ARCHIVE_SUFFIXES = {'.txt', '.html', '.json', '.png', '.jpg', '.jpeg'}
def is_archive_file(path: pathlib.Path) -> bool:
"""
Archive files are expected to start with the following: %Y-%m-%d-%H-%M-%S
they must have one of the following suffixes: .txt, .html, .json, .png, .jpg, .jpeg
"""
return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(ARCHIVE_MATCHER.match(path.name))
SINGLEFILE_HEADER = '''<!--
Page saved with SingleFile'''
@ -492,7 +344,7 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
# A file name matching exactly the Archive format from WROLPi should always be an Archive.
return True
with path.open('rt') as fh:
# Lastly, try to read the header of this HTML file.
# Lastly, try to read the header of this HTML file. Read a large part of the head because URLs can be long.
header = fh.read(1000)
if SINGLEFILE_HEADER in header:
return True
@ -514,68 +366,64 @@ def get_domains():
ARCHIVE_ORDERS = {
'date': 'a.archive_datetime ASC, LOWER(a.singlefile_path) ASC',
'-date': 'a.archive_datetime DESC NULLS LAST, LOWER(a.singlefile_path) DESC',
'date': 'a.archive_datetime ASC, LOWER(fg.primary_path) ASC',
'-date': 'a.archive_datetime DESC NULLS LAST, LOWER(fg.primary_path) DESC',
'rank': '2 DESC, a.archive_datetime DESC',
'-rank': '2 ASC, a.archive_datetime ASC',
}
def archive_search(search_str: str, domain: str, limit: int, offset: int, order_by: str) -> Tuple[List[Archive], int]:
wheres = []
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str]) \
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = ["fg.model = 'archive'"]
joins = []
params = dict(search_str=search_str, offset=int(offset), limit=int(limit))
order = '1 DESC'
order_by = '1 DESC'
if search_str:
# A search_str was provided by the user, modify the query to filter by it.
select_columns = 'f.path, ts_rank(f.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
wheres.append('f.textsearch @@ websearch_to_tsquery(%(search_str)s)')
select_columns = 'ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
params['search_str'] = search_str
join = 'LEFT JOIN file f on f.path = a.singlefile_path'
else:
# No search_str provided. Get path and total only. Don't join the "file" to speed up query.
select_columns = 'a.singlefile_path AS path, COUNT(*) OVER() AS total'
join = ''
select_columns = 'fg.id, COUNT(*) OVER() AS total'
if order_by:
if order:
try:
order = ARCHIVE_ORDERS[order_by]
order_by = ARCHIVE_ORDERS[order]
except KeyError:
raise InvalidOrderBy(f'Invalid order by: {order_by}')
raise InvalidOrderBy(f'Invalid order by: {order}')
if tag_names:
where_, params_, join_ = tag_names_to_clauses(tag_names)
wheres.append(where_)
params.update(params_)
joins.append(join_)
if domain:
params['domain'] = domain
wheres.append('domain_id = (select id from domains where domains.domain = %(domain)s)')
wheres.append('a.domain_id = (select id from domains where domains.domain = %(domain)s)')
select_columns = f", {select_columns}" if select_columns else ""
wheres = '\n AND '.join(wheres)
where = f'WHERE\n{wheres}' if wheres else ''
join = '\n'.join(joins)
stmt = f'''
SELECT
{select_columns}
FROM archive a
fg.id -- always get `file_group.id` for `handle_file_group_search_results`
{select_columns}
FROM file_group fg
LEFT JOIN archive a ON a.file_group_id = fg.id
{join}
{where}
ORDER BY {order}
GROUP BY fg.id, a.archive_datetime
ORDER BY {order_by}
OFFSET %(offset)s LIMIT %(limit)s
'''.strip()
logger.debug(stmt, params)
results, total = handle_search_results(stmt, params)
results, total = handle_file_group_search_results(stmt, params)
return results, total
match_archive_paths = partial(match_paths_to_suffixes, suffix_groups=(
('.readability.html',),
('.html',),
('.readability.json',),
('.readability.txt',),
('.jpg', '.jpeg', '.webp', '.png'),
))
@iterify(tuple)
def match_archive_files(files: List[File]) -> Tuple[File, File, File, File, File]:
archive_paths = match_archive_paths([i.path for i in files])
for path in archive_paths:
yield next((i for i in files if i.path == path), None)

View file

@ -2,19 +2,19 @@ import datetime
import json
import pathlib
import re
from typing import Generator, Optional, Iterable
from typing import Iterable, List, Optional
import pytz
from sqlalchemy import Column, Integer, String, ForeignKey, Boolean
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
from sqlalchemy.orm import relationship, Session, validates
from sqlalchemy.orm.collections import InstrumentedList
from wrolpi.common import ModelHelper, Base, logger
from wrolpi.dates import TZDateTime
from wrolpi.errors import InvalidArchive
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.files.models import File
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@ -28,25 +28,13 @@ class Archive(Base, ModelHelper):
__tablename__ = 'archive'
id = Column(Integer, primary_key=True)
archive_datetime = Column(TZDateTime)
url = Column(String)
domain_id = Column(Integer, ForeignKey('domains.id'))
domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id')
url = Column(String)
title = Column(String)
archive_datetime = Column(TZDateTime)
validated = Column(Boolean)
# Associated Files.
singlefile_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
singlefile_file: File = relationship('File', primaryjoin='Archive.singlefile_path==File.path')
readability_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
readability_file: File = relationship('File', primaryjoin='Archive.readability_path==File.path')
readability_json_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
readability_json_file: File = relationship('File', primaryjoin='Archive.readability_json_path==File.path')
readability_txt_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
readability_txt_file: File = relationship('File', primaryjoin='Archive.readability_txt_path==File.path')
screenshot_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
screenshot_file: File = relationship('File', primaryjoin='Archive.screenshot_path==File.path')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
file_group: FileGroup = relationship('FileGroup')
def __repr__(self):
if self.domain:
@ -54,29 +42,11 @@ class Archive(Base, ModelHelper):
f'domain={self.domain.domain}>'
return f'<Archive id={self.id} url={self.url} singlefile={repr(str(self.singlefile_path))}>'
def my_paths(self) -> Generator[pathlib.Path, None, None]:
if self.singlefile_path:
yield self.singlefile_path
if self.readability_path:
yield self.readability_path
if self.readability_json_path:
yield self.readability_json_path
if self.readability_txt_path:
yield self.readability_txt_path
if self.screenshot_path:
yield self.screenshot_path
def my_paths(self, *mimetypes: str) -> List[pathlib.Path]:
return self.file_group.my_paths(*mimetypes)
def my_files(self) -> Generator[File, None, None]:
if self.singlefile_file:
yield self.singlefile_file
if self.readability_file:
yield self.readability_file
if self.readability_json_file:
yield self.readability_json_file
if self.readability_txt_file:
yield self.readability_txt_file
if self.screenshot_file:
yield self.screenshot_file
def my_files(self, *mimetypes: str) -> List[dict]:
return self.file_group.my_files(*mimetypes)
def unlink(self):
"""
@ -95,32 +65,12 @@ class Archive(Base, ModelHelper):
return self.archive_datetime > other.archive_datetime
def __json__(self):
stem, _ = split_path_stem_and_suffix(self.singlefile_path) if self.singlefile_path else (None, None)
d = dict(
archive_datetime=self.archive_datetime,
domain=self.domain.dict() if self.domain else None,
domain_id=self.domain_id,
id=self.id,
readability_json_path=self.readability_json_path,
readability_path=self.readability_path,
readability_txt_path=self.readability_txt_path,
screenshot_path=self.screenshot_path,
singlefile_path=self.singlefile_path,
stem=stem,
title=self.title,
url=self.url,
)
return d
def delete(self):
self.unlink()
session = Session.object_session(self)
for file in self.my_files():
session.delete(file)
session.delete(self.file_group)
session.delete(self)
if self.domain:
@ -139,101 +89,175 @@ class Archive(Base, ModelHelper):
).order_by(Archive.archive_datetime))
return history
def read_singlefile_data(self):
"""
Read the start of the singlefile (if any) and decode the Archive information.
"""
path = self.singlefile_file.path if self.singlefile_file else self.singlefile_path
if not path:
# Can't read contents of nothing.
raise ValueError('Cannot read singlefile data when this has no files')
@property
def singlefile_file(self) -> Optional[dict]:
from modules.archive.lib import is_singlefile_file
files = self.file_group.my_files('text/html')
for file in files:
if is_singlefile_file(file['path']):
return file
with path.open('rt') as fh:
head = fh.read(1000)
if 'Page saved with SingleFile' not in head:
return
try:
if match := MATCH_URL.findall(head):
self.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
try:
if match := MATCH_DATE.findall(head):
dt = match[0].strip()
dt = ' '.join(dt.split(' ')[:5])
# SingleFile uses GMT.
dt = datetime.datetime.strptime(
dt,
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
).replace(tzinfo=pytz.timezone('GMT'))
self.archive_datetime = dt
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
@property
def singlefile_path(self) -> Optional[pathlib.Path]:
if singlefile_file := self.singlefile_file:
return singlefile_file['path']
@property
def readability_file(self) -> Optional[dict]:
files = self.file_group.my_files('text/html')
for file in files:
if file['path'].name.endswith('.readability.html'):
return file
@property
def readability_path(self) -> Optional[pathlib.Path]:
if readability_file := self.readability_file:
return readability_file['path']
@property
def readability_txt_file(self) -> Optional[dict]:
files = self.file_group.my_files('text/plain', 'text/html')
for file in files:
if file['path'].name.endswith('.readability.txt'):
return file
@property
def readability_txt_path(self) -> Optional[pathlib.Path]:
if readability_txt_file := self.readability_txt_file:
return readability_txt_file['path']
@property
def readability_json_file(self) -> Optional[dict]:
files = self.file_group.my_files()
for file in files:
if file['path'].name.endswith('.readability.json'):
return file
@property
def readability_json_path(self) -> Optional[pathlib.Path]:
if readability_json_file := self.readability_json_file:
return readability_json_file['path']
@property
def screenshot_file(self) -> Optional[dict]:
files = self.file_group.my_files('image/')
for file in files:
return file
@property
def screenshot_path(self) -> Optional[pathlib.Path]:
if screenshot_file := self.screenshot_file:
return screenshot_file['path']
def read_readability_data(self):
"""Read the Readability JSON file, apply its contents to this record."""
readability_json_file = self.readability_json_file
if not readability_json_file or not readability_json_file.path.is_file():
path = self.singlefile_file.path if self.singlefile_file else self.singlefile_path
logger.warning(f'{path} does not have an info json file')
readability_json_path = self.readability_json_path
if not readability_json_path:
logger.warning(f'{self.singlefile_path} does not have an info json file')
return
if not readability_json_path.is_file():
error = f'Cannot read data from {readability_json_path} because it not exist.'
if PYTEST:
raise ValueError(error)
logger.error(error)
return
try:
with readability_json_file.path.open() as fh:
with readability_json_path.open() as fh:
json_contents = json.load(fh)
url = json_contents.get('url')
title = json_contents.get('title')
except Exception as e:
raise InvalidArchive() from e
self.url = url
self.title = title
# Readability is most trusted, it should overwrite any previous data.
self.url = url or self.url
self.file_group.title = title or self.file_group.title
def read_singlefile_data(self):
"""Read the start of the singlefile (if any) and extract any Archive information."""
path = self.singlefile_path
if not path:
# Can't read contents of nothing.
raise ValueError('Cannot read singlefile data when this has no files')
if self.url and self.archive_datetime:
# This data has already been read.
return
with path.open('rt') as fh:
head = fh.read(1000)
if 'Page saved with SingleFile' not in head:
logger.error(f'Could not find SingleFile header in {self}')
return
if not self.url:
try:
if match := MATCH_URL.findall(head):
self.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
if not self.archive_datetime:
try:
if match := MATCH_DATE.findall(head):
dt = match[0].strip()
dt = ' '.join(dt.split(' ')[:5])
# SingleFile uses GMT.
dt = datetime.datetime.strptime(
dt,
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
).replace(tzinfo=pytz.timezone('GMT'))
self.archive_datetime = dt
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
def apply_domain(self):
"""Get the domain from the URL."""
from modules.archive.lib import get_or_create_domain
domain = None
if self.url:
domain = get_or_create_domain(Session.object_session(self), self.url)
session = Session.object_session(self)
if not session:
raise ValueError('No session found!')
domain = get_or_create_domain(session, self.url)
# Clear domain if the URL is missing.
self.domain_id = domain.id if domain else None
def apply_title(self):
def apply_singlefile_title(self):
"""Get the title from the Singlefile, if its missing."""
from modules.archive.lib import get_title_from_html
if self.singlefile_path and not self.title:
self.title = get_title_from_html(self.singlefile_path.read_text())
if self.singlefile_path and not self.file_group.title:
self.file_group.title = get_title_from_html(self.singlefile_path.read_text())
def validate(self):
"""Fill in any missing data about this Archive from it's files."""
try:
self.read_readability_data()
self.read_singlefile_data()
self.apply_domain()
self.apply_title()
self.validated = True
self.apply_singlefile_title()
except Exception as e:
logger.warning(f'Unable to validate {self}', exc_info=e)
if PYTEST:
raise
@staticmethod
def find_by_path(path, session: Session) -> Optional[Base]:
archive = session.query(Archive).filter_by(singlefile_path=path).one_or_none()
def from_paths(session: Session, *paths: pathlib.Path) -> 'Archive':
"""Create a new Archive and FileGroup from the provided paths.
The files will be read and Archive data extracted."""
from modules.archive import model_archive
file_group = FileGroup.from_paths(session, *paths)
archive = model_archive(file_group, session)
return archive
@staticmethod
def find_by_paths(paths, session):
archives = list(session.query(Archive).filter(Archive.singlefile_path.in_(paths)))
return archives
@property
def primary_path(self):
return self.singlefile_path
class Domain(Base, ModelHelper):
__tablename__ = 'domains' # plural to avoid conflict
id = Column(Integer, primary_key=True)
domain = Column(String)
domain = Column(String, nullable=False)
directory = Column(MediaPathType)
archives: InstrumentedList = relationship('Archive', primaryjoin='Archive.domain_id==Domain.id')

View file

@ -1,4 +1,4 @@
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime
from typing import List, Optional
@ -30,9 +30,10 @@ class ArchiveSearchRequest:
offset: Optional[int] = None
limit: Optional[int] = None
order_by: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
@dataclass
class ArchiveSearchResponse:
videos: List[ArchiveDict]
file_groups: List[ArchiveDict]
totals: dict

View file

@ -4,11 +4,14 @@ from http import HTTPStatus
def check_results(test_client, data, ids):
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK, response.json
if not response.status_code == HTTPStatus.OK:
raise AssertionError(str(response.json))
if ids:
assert response.json['archives'], f'Expected {len(ids)} archives but did not receive any'
assert [i['archive']['id'] for i in response.json['archives']] == ids, \
f'{response.json["archives"][0]["archive"]["id"]}..{response.json["archives"]["archive"][19]["id"]}'
file_groups = response.json['file_groups']
assert file_groups, f'Expected {len(ids)} archives but did not receive any'
assert [i['id'] for i in response.json['file_groups']] == ids, \
f'{response.json["file_groups"][0]["id"]}..{response.json["file_groups"][-1]["id"]}'
def test_archives_search(test_session, archive_directory, archive_factory, test_client):
@ -87,22 +90,22 @@ def test_archives_search_no_query(test_session, archive_factory, test_client):
# All archives are returned when no `search_str` is passed.
request, response = test_client.post('/api/archive/search', content='{}')
assert response.status_code == HTTPStatus.OK, response.json
assert [i['archive']['id'] for i in response.json['archives']] == list(range(100, 80, -1))
assert response.json['totals']['archives'] == 100
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
assert response.json['totals']['file_groups'] == 100
# All archives are from "example.com".
data = dict(domain='example.com')
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK, response.json
assert [i['archive']['id'] for i in response.json['archives']] == list(range(100, 80, -1))
assert response.json['totals']['archives'] == 100
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
assert response.json['totals']['file_groups'] == 100
# No archives are from "example.org".
data = dict(domain='example.org')
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK, response.json
assert [i['archive']['id'] for i in response.json['archives']] == []
assert response.json['totals']['archives'] == 0
assert [i['id'] for i in response.json['file_groups']] == []
assert response.json['totals']['file_groups'] == 0
def test_archive_and_domain_crud(test_session, test_client, archive_factory):
@ -121,14 +124,14 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
# Archive1 has Archive2 as history.
request, response = test_client.get(f'/api/archive/{archive1.id}')
assert response.status_code == HTTPStatus.OK
assert response.json['file']['archive']['id'] == archive1.id
assert response.json['history'][0]['archive']['id'] == archive2.id
assert response.json['file_group']['id'] == archive1.id
assert response.json['history'][0]['id'] == archive2.id
# Archive2 has Archive1 as history.
request, response = test_client.get(f'/api/archive/{archive2.id}')
assert response.status_code == HTTPStatus.OK
assert archive2.id == response.json['file']['archive']['id']
assert response.json['history'][0]['archive']['id'] == archive1.id
assert archive2.id == response.json['file_group']['id']
assert response.json['history'][0]['id'] == archive1.id
# Only one domain.
request, response = test_client.get(f'/api/archive/domains')
@ -150,7 +153,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
# Archive2 no longer has history.
request, response = test_client.get(f'/api/archive/{archive2.id}')
assert response.status_code == HTTPStatus.OK
assert response.json['file']['archive']['id'] == archive2.id
assert response.json['file_group']['id'] == archive2.id
assert response.json['history'] == []
# No Archives, no Domains.

View file

@ -1,31 +1,36 @@
import json
import pathlib
import tempfile
from datetime import datetime, timedelta
from http import HTTPStatus
import mock
import pytest
from PIL import Image
from modules.archive import lib
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, do_archive, get_domains, \
group_archive_files, ArchiveFiles
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, do_archive, get_domains
from modules.archive.models import Archive, Domain
from wrolpi.db import get_db_session
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import File
from wrolpi.files.models import FileGroup
from wrolpi.root_api import CustomJSONEncoder
from wrolpi.test.common import skip_circleci
def make_fake_request_archive(readability=True, screenshot=True, title=True):
async def fake_request_archive(_):
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
Image.new('RGB', (1, 1), color='grey').save(fh)
fh.seek(0)
image = fh.read()
singlefile = '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
r = dict(
content=f'<html>test readability content</html>',
textContent='<html>test readability textContent</html>',
textContent='test readability textContent',
title='ジにてこちら' if title else None,
) if readability else None
s = b'screenshot data' if screenshot else None
s = image if screenshot else None
return singlefile, r, s
return fake_request_archive
@ -61,13 +66,12 @@ async def test_dict(test_session):
@pytest.mark.asyncio
async def test_relationships(test_session, test_directory):
async def test_relationships(test_session, test_directory, example_singlefile):
with get_db_session(commit=True) as session:
url = 'https://wrolpi.org:443'
domain = get_or_create_domain(session, url)
archive = Archive(
singlefile_file=File(path=test_directory / 'wrolpi.org:443/foo'),
title='bar',
file_group=FileGroup.from_paths(test_session, example_singlefile),
url=url,
domain_id=domain.id,
)
@ -77,6 +81,50 @@ async def test_relationships(test_session, test_directory):
assert archive.domain == domain
@pytest.mark.asyncio
async def test_archive_title(test_session, archive_factory, singlefile_contents_factory):
"""An Archive's title can be fetched in multiple ways. This tests from most to least reliable."""
# Create some test files, delete all records for a fresh refresh.
archive_factory(
domain='example.com',
url='https://example.com/json-url',
title='json title',
singlefile_contents=singlefile_contents_factory('singlefile title', 'https://example.com/singlefile-url'),
)
test_session.query(FileGroup).delete()
test_session.commit()
await refresh_files()
assert test_session.query(FileGroup).count() == 1
assert test_session.query(Archive).count() == 1
archive1: Archive = test_session.query(Archive).one()
# Readability JSON is trusted first.
assert archive1.url == 'https://example.com/json-url'
assert archive1.file_group.title == 'json title'
# Delete JSON file.
archive1.readability_json_path.unlink()
test_session.delete(archive1)
await refresh_files()
archive1: Archive = test_session.query(Archive).one()
assert archive1.url == 'https://example.com/singlefile-url'
assert archive1.file_group.title == 'singlefile title'
# Clear out title and URL from Singlefile.
archive1.singlefile_path.write_text(singlefile_contents_factory('', ''))
test_session.delete(archive1)
await refresh_files()
# No title/URL could be found.
archive1: Archive = test_session.query(Archive).one()
assert archive1.url is None
assert archive1.file_group.title is None
def test_archive_refresh_deleted_archive(test_client, test_session, archive_directory, archive_factory):
"""Archives/Domains should be deleted when archive files are deleted."""
archive1 = archive_factory('example.com', 'https://example.com/1')
@ -125,17 +173,17 @@ def test_archive_refresh_deleted_archive(test_client, test_session, archive_dire
check_counts(archive_count=0, domain_count=0)
def test_fills_contents_with_refresh(test_client, test_session, archive_factory):
@pytest.mark.asyncio
async def test_fills_contents_with_refresh(test_client, test_session, archive_factory, singlefile_contents_factory):
"""Refreshing archives fills in any missing contents."""
archive1 = archive_factory('example.com', 'https://example.com/one')
archive2 = archive_factory('example.com', 'https://example.com/one')
# Title can be found in archive3's singlefile
archive3 = archive_factory('example.org',
singlefile_contents='<html> some stuff<title>last title</title> other stuff</html>')
archive3 = archive_factory('example.org', singlefile_contents=singlefile_contents_factory('last title'))
archive4 = archive_factory('example.org') # has no contents
# Clear the title, this will be filled in.
archive1.title = archive2.title = archive3.title = archive4.title = None
archive1.file_group.title = archive2.file_group.title = archive3.file_group.title = archive4.file_group.title = None
contents_title_archive = [
('foo bar qux', 'my archive', archive1),
@ -143,31 +191,31 @@ def test_fills_contents_with_refresh(test_client, test_session, archive_factory)
('baz qux qux qux', None, archive3),
]
for contents, json_title, archive in contents_title_archive:
with archive.readability_txt_path.open('wt') as fh:
archive: Archive
readability_txt_path = archive.singlefile_path.with_suffix('.readability.txt')
with readability_txt_path.open('wt') as fh:
fh.write(contents)
if json_title:
archive.readability_json_path.write_text(json.dumps({'title': json_title, 'url': archive.url}))
readability_json_path = archive.singlefile_path.with_suffix('.readability.json')
readability_json_path.write_text(json.dumps({'title': json_title, 'url': archive.url}))
test_session.commit()
# Contents are empty. Archives are all missing the title.
assert not archive1.singlefile_file.d_text
assert not archive2.singlefile_file.d_text
assert not archive3.singlefile_file.d_text
assert not archive4.singlefile_file.d_text
assert not archive1.file_group.d_text
assert not archive2.file_group.d_text
assert not archive3.file_group.d_text
assert not archive4.file_group.d_text
# Fill the contents.
test_client.post('/api/files/refresh')
await refresh_files()
# The archives will be renamed with their title.
archive1, archive2, archive3, archive4 = test_session.query(Archive).order_by(Archive.id)
assert archive1.singlefile_file.title
assert archive2.singlefile_file.title
assert archive3.singlefile_file.title
assert not archive4.singlefile_file.d_text
assert not archive4.file_group.d_text
# Missing title is filled.
assert archive1.title == 'my archive'
assert archive2.title == 'other archive'
assert archive3.title == 'last title' # from json
# Missing titles are filled.
assert archive1.file_group.title == 'my archive'
assert archive2.file_group.title == 'other archive'
assert archive3.file_group.title == 'last title' # from singlefile HTML
def test_delete_archive(test_session, archive_factory):
@ -183,12 +231,12 @@ def test_delete_archive(test_session, archive_factory):
assert archive3.my_paths() and all(i.is_file() for i in archive3.my_paths())
assert test_session.query(Archive).count() == 3
assert test_session.query(File).count() == 15
assert test_session.query(FileGroup).count() == 3
# Delete the oldest.
delete_archives(archive1.id, archive3.id)
assert test_session.query(Archive).count() == 1
assert test_session.query(File).count() == 5
assert test_session.query(FileGroup).count() == 1
# Files were deleted.
assert archive1.my_paths() and not any(i.is_file() for i in archive1.my_paths())
assert archive3.my_paths() and not any(i.is_file() for i in archive3.my_paths())
@ -202,7 +250,7 @@ def test_delete_archive(test_session, archive_factory):
assert domain is None
# All Files were deleted.
assert test_session.query(File).count() == 0
assert test_session.query(FileGroup).count() == 0
assert archive2.my_paths() and not any(i.is_file() for i in archive2.my_paths())
@ -215,6 +263,10 @@ def test_get_domains(test_session, archive_factory):
archive3 = archive_factory('example.org')
test_session.commit()
assert archive1.domain
assert archive2.domain
assert archive3.domain
assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org']
archive2.delete()
@ -242,17 +294,17 @@ async def test_new_archive(test_session, fake_now):
assert isinstance(archive1.readability_path, pathlib.Path)
assert isinstance(archive1.readability_txt_path, pathlib.Path)
assert isinstance(archive1.screenshot_path, pathlib.Path)
assert archive1.title == 'ジにてこちら'
assert archive1.file_group.title == 'ジにてこちら'
assert archive1.url is not None
assert archive1.domain is not None
# The actual files were dumped and read correctly.
with open(archive1.singlefile_path) as fh:
assert fh.read() == '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
assert fh.read() == '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
with open(archive1.readability_path) as fh:
assert fh.read() == '<html>test readability content</html>'
with open(archive1.readability_txt_path) as fh:
assert fh.read() == '<html>test readability textContent</html>'
assert fh.read() == 'test readability textContent'
with open(archive1.readability_json_path) as fh:
assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
@ -267,7 +319,7 @@ async def test_get_title_from_html(test_session, fake_now):
fake_now(datetime(2000, 1, 1))
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive = await do_archive('https://example.com')
assert archive.title == 'ジにてこちら'
assert archive.file_group.title == 'ジにてこちら'
async def fake_request_archive(_):
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
@ -281,7 +333,7 @@ async def test_get_title_from_html(test_session, fake_now):
fake_now(datetime(2000, 1, 2))
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = await do_archive('https://example.com')
assert archive.title == 'some title'
assert archive.file_group.title == 'some title'
async def fake_request_archive(_):
singlefile = '<html></html>'
@ -295,7 +347,7 @@ async def test_get_title_from_html(test_session, fake_now):
fake_now(datetime(2000, 1, 3))
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = await do_archive('https://example.com')
assert archive.title is None
assert archive.file_group.title is None
@skip_circleci
@ -333,7 +385,7 @@ def test_get_new_archive_files(fake_now):
@skip_circleci
@pytest.mark.asyncio
async def test_title_in_filename(test_session, fake_now, test_directory):
async def test_title_in_filename(test_session, fake_now, test_directory, image_bytes_factory):
"""
The Archive files have the title in the path.
"""
@ -342,7 +394,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive1 = await do_archive('https://example.com')
assert archive1.title == 'ジにてこちら'
assert archive1.file_group.title == 'ジにてこちら'
assert str(archive1.singlefile_path.relative_to(test_directory)) == \
'archive/example.com/2000-01-01-00-00-00_ジにてこちら.html'
@ -360,9 +412,9 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
r = dict(
# No Title from Readability.
content=f'<html>test readability content</html>',
textContent='<html>test readability textContent</html>',
textContent='test readability textContent',
)
s = b'screenshot data'
s = image_bytes_factory()
return singlefile, r, s
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
@ -386,7 +438,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
content=f'<html>test readability content</html>',
textContent='<html>test readability textContent</html>',
)
s = b'screenshot data'
s = image_bytes_factory()
return singlefile, r, s
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
@ -404,7 +456,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.png'
@skip_circleci
# @skip_circleci
def test_refresh_archives(test_session, test_directory, test_client, make_files_structure):
"""
Archives can be found and put in the database. A single Archive will have multiple files.
@ -418,11 +470,11 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
make_files_structure({
# These are all for an individual Archive.
'archive/example.com/2021-10-05 16:20:10.346823.html': singlefile_contents, # renames to 2021-10-05-16-20-10_NA
'archive/example.com/2021-10-05 16:20:10.346823.png': None,
'archive/example.com/2021-10-05 16:20:10.346823-readability.txt': 'article text contents',
'archive/example.com/2021-10-05 16:20:10.346823-readability.json': '{"url": "https://example.com"}',
'archive/example.com/2021-10-05 16:20:10.346823-readability.html': '<html></html>',
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
'archive/example.com/2021-10-05-16-20-10_NA.png': None,
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents',
'archive/example.com/2021-10-05-16-20-10_NA.readability.json': '{"url": "https://example.com"}',
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
# These should log an error because they are missing the singlefile path.
'archive/example.com/2021-10-05 16:20:10.readability.html': '<html></html>',
'archive/example.com/2021-10-05 16:20:10.readability.json': None,
@ -435,6 +487,11 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
test_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
# Cause a re-index of the archive.
test_session.query(FileGroup).filter_by(
full_stem=str(test_directory / 'archive/example.com/2021-10-05-16-20-10_NA')).one().indexed = False
test_session.commit()
# Running the refresh does not result in a new archive.
test_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
@ -452,11 +509,15 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
content = json.dumps({'search_str': 'text', 'model': 'archive'})
request, response = test_client.post('/api/files/search', content=content)
assert response.status_code == HTTPStatus.OK
assert response.json['files'], 'No files matched "text"'
assert response.json['files'][0]['model'] == 'archive', 'Returned file was not an archive'
assert response.json['files'][0].get('path') == 'archive/example.com/2021-10-05-16-20-10_NA.html', \
assert response.json['file_groups'], 'No files matched "text"'
assert response.json['file_groups'][0]['model'] == 'archive', 'Returned file was not an archive'
assert response.json['file_groups'][0]['data'].get('readability_path') == \
'archive/example.com/2021-10-05-16-20-10_NA.readability.html',\
'Could not find readability html file'
assert response.json['file_groups'][0]['data'].get('readability_txt_path') == \
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt',\
'Could not find readability text file containing "text"'
assert response.json['files'][0]['archive']['archive_datetime'], 'Archive has no datetime'
assert response.json['file_groups'][0]['data']['archive_datetime'], 'Archive has no datetime'
@pytest.mark.asyncio
@ -482,18 +543,9 @@ async def test_refresh_archives_index(test_session, test_directory, test_client,
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
assert archive.singlefile_file.a_text == 'the title'
assert archive.singlefile_file.d_text == 'article text contents'
# The associated files are not indexed.
for name in ('screenshot_file', 'readability_file', 'readability_json_file', 'readability_txt_file'):
file = getattr(archive, name)
assert file.associated is True, 'The associated file is not marked'
assert file.indexed is False
# file.a_text is the file name.
assert not file.b_text
assert not file.c_text
assert not file.d_text
assert archive.file_group.a_text == 'the title'
assert archive.file_group.d_text == 'article text contents'
assert archive.file_group.indexed is True
def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure):
@ -506,76 +558,24 @@ def test_refresh_archives_invalid_file(test_session, test_directory, test_client
'bogus file': None,
})
test_client.post('/api/files/refresh')
assert test_session.query(File).count() == 3 # Three files total.
assert test_session.query(FileGroup).count() == 2 # One bogus file, one archive group.
assert test_session.query(Archive).count() == 1 # Only one real archive.
# Create an invalid Archive, it should be removed during refresh.
test_session.add(Archive(singlefile_path=bogus_path))
# Claim the bogus file as an Archive, this should be removed.
bogus_file: File = test_session.query(File).filter_by(path=bogus_path).one()
bogus_file: FileGroup = test_session.query(FileGroup).filter_by(primary_path=bogus_path).one()
bogus_file.model = 'archive'
archive = Archive(file_group=bogus_file)
test_session.add(archive)
test_session.commit()
assert test_session.query(File).count() == 3
assert test_session.query(FileGroup).count() == 2
assert test_session.query(Archive).count() == 2
test_client.post('/api/files/refresh')
assert test_session.query(File).count() == 3, 'More files in DB than we created'
assert test_session.query(FileGroup).count() == 2, 'More files in DB than we created'
assert test_session.query(Archive).count() == 1, 'The bogus Archive was not removed'
assert bogus_file.model is None, 'Model was not removed'
def test_group_archive_files(test_directory):
"""Archive files should be grouped together when they are the same Archive."""
files = [
pathlib.Path('2021-10-05 16:20:10.346823.html'),
pathlib.Path('2021-10-05 16:20:10.346823.readability.json'),
pathlib.Path('2000-01-01-00-00-00_Title.html'),
pathlib.Path('2000-01-01-00-00-00_Title.readability.json'),
pathlib.Path('not an archive'),
pathlib.Path('2000-01-01-00-00-01_Missing a singlefile.readability.json'),
]
group1 = ArchiveFiles(
singlefile=pathlib.Path('2000-01-01-00-00-00_Title.html'),
readability_json=pathlib.Path('2000-01-01-00-00-00_Title.readability.json')
)
group2 = ArchiveFiles(
singlefile=pathlib.Path('2021-10-05 16:20:10.346823.html'),
readability_json=pathlib.Path('2021-10-05 16:20:10.346823.readability.json'),
)
assert list(group_archive_files(files)) == [
(datetime(2000, 1, 1, 0, 0, 0), group1),
(datetime(2021, 10, 5, 16, 20, 10), group2),
]
@pytest.mark.parametrize(
'name,expected', [
('foo', False),
('2000-01-01-00-00-00_Some Title.html', True),
('2000-01-01-00-00-00_Some Title.readability.json', True),
('2000-01-01-00-00-00_Some Title.readability.html', True),
('2000-01-01-00-00-00_Some Title.readability.txt', True),
('2000-01-01-00-00-00_Some Title.png', True),
('2000-01-01-00-00-00_Some Title.jpg', True),
('2000-01-01-00-00-00_Some Title.jpeg', True),
('2000-01-01-00-00-00_Some NA.html', True),
('2000-01-01-00-00-00_NA.readability.json', True),
('2000-01-01-00-00-00_NA.readability.html', True),
('2000-01-01-00-00-00_NA.readability.txt', True),
('2000-01-01-00-00-00_NA.png', True),
('2000-01-01-00-00-00_NA.jpg', True),
('2000-01-01-00-00-00_NA.jpeg', True),
('2000-01-01-00-00-00_ジにてこちら.html', True),
('2000-01-01-00-00-00no underscore.html', False),
]
)
def test_is_archive_file(name, expected, test_directory):
path = test_directory / name
path.touch()
assert lib.is_archive_file(path) == expected
@pytest.mark.parametrize(
'name,expected', [
('foo', False),
@ -587,69 +587,12 @@ def test_is_archive_file(name, expected, test_directory):
('foo.html', True),
]
)
def test_is_singlefile_file(name, expected, make_files_structure, singlefile_contents):
def test_is_singlefile_file(name, expected, make_files_structure, singlefile_contents_factory):
"""A Singlefile may not be created by WROLPi, an HTML file created by Singlefile can also be an Archive."""
path, = make_files_structure({name: singlefile_contents})
path, = make_files_structure({name: singlefile_contents_factory()})
assert lib.is_singlefile_file(path) == expected
@skip_circleci
def test_migrate_archive_files(test_session, archive_directory, archive_factory):
"""Test that migration of all old Archive formatted files are migrated."""
# Create some updated archives that should be ignored.
archive1 = archive_factory('example.com', title='the title')
archive2 = archive_factory('example.com')
archive3 = archive_factory('example.org')
test_session.commit()
assert archive1.singlefile_path.is_file()
assert archive2.singlefile_path.is_file()
assert archive3.singlefile_path.is_file()
example_com_directory = archive_directory / 'example.com'
example_org_directory = archive_directory / 'example.org'
# Archive4
(example_com_directory / '2021-10-05 16:20:10.346823.html').touch()
path = example_com_directory / '2021-10-05 16:20:10.346823.readability.json'
path.write_text(json.dumps({'title': 'my title'}))
# Archive5
(example_com_directory / '2000-01-01 00:00:00.000000.html').touch()
path = example_com_directory / '2000-01-01 00:00:00.000000.readability.json'
path.write_text(json.dumps({'title': 'invalid#*: title'}))
# Archive6
(example_org_directory / '2000-01-01 00:00:00.000000.html').touch()
path = example_org_directory / '2000-01-01 00:00:00.000000.readability.json'
path.write_text(json.dumps({'title': 'A' * 500})) # Really long file name.
# Migration forms a plan, then performs it.
plan = lib.migrate_archive_files()
for old, new in plan:
# All files are moved.
assert not old.is_file()
assert new.is_file()
# Check that only the outdated files are moved.
plan = [(i.relative_to(archive_directory), j.relative_to(archive_directory)) for i, j in plan]
assert plan == [
(pathlib.Path('example.com/2000-01-01 00:00:00.000000.html'),
pathlib.Path('example.com/2000-01-01-00-00-00_invalid# title.html')),
(pathlib.Path('example.com/2000-01-01 00:00:00.000000.readability.json'),
pathlib.Path('example.com/2000-01-01-00-00-00_invalid# title.readability.json')),
(pathlib.Path('example.com/2021-10-05 16:20:10.346823.html'),
pathlib.Path('example.com/2021-10-05-16-20-10_my title.html')),
(pathlib.Path('example.com/2021-10-05 16:20:10.346823.readability.json'),
pathlib.Path('example.com/2021-10-05-16-20-10_my title.readability.json')),
(pathlib.Path('example.org/2000-01-01 00:00:00.000000.html'),
pathlib.Path('example.org/2000-01-01-00-00-00_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.html')),
(pathlib.Path('example.org/2000-01-01 00:00:00.000000.readability.json'),
pathlib.Path(
'example.org/2000-01-01-00-00-00_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.readability.json')),
]
# New Archive files still exist.
assert all(i.is_file() for i in archive1.my_paths())
assert all(i.is_file() for i in archive2.my_paths())
assert all(i.is_file() for i in archive3.my_paths())
def test_archive_files(test_directory):
af = lib.ArchiveFiles(
singlefile=test_directory / 'archive/singlefile.html',
@ -687,17 +630,11 @@ async def test_archive_download_index(test_session, test_directory, image_file):
assert archive.readability_path and archive.readability_path.exists()
assert archive.readability_json_path
assert archive.readability_txt_path and archive.readability_txt_path.exists()
assert archive.readability_txt_file.d_text == 'the readability', 'Readability text was not indexed'
assert archive.title == 'the singlefile', 'Did not get the title from the singlefile'
assert archive.file_group.d_text == 'the readability', 'Readability text was not indexed'
assert archive.file_group.title == 'the singlefile', 'Did not get the title from the singlefile'
assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \
'Did not store the screenshot'
assert not archive.singlefile_file.associated
assert archive.readability_txt_file.associated \
and archive.readability_json_file.associated \
and archive.readability_file.associated \
and archive.screenshot_file.associated, 'Archive files must be associated'
def test_archive_history(test_session, archive_factory):
"""Archive's can have a history of other archives.

View file

@ -1,163 +1,89 @@
import json
import pathlib
from abc import ABC
from typing import List, Dict
from sqlalchemy.orm import Session
import asyncio
from typing import List, Tuple
from modules.videos.models import Video, Channel
from wrolpi.captions import read_captions, extract_captions
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
from wrolpi.db import get_db_curs
from wrolpi.files.indexers import Indexer, register_indexer
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.files.models import File
from wrolpi.common import logger, limit_concurrent, register_modeler, register_after_refresh
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
from .downloader import video_downloader # Import downloaders so they are registered.
logger = logger.getChild(__name__)
__all__ = ['video_modeler', 'VideoIndexer']
__all__ = ['video_modeler']
def find_video_file_in_group(group: List[File]):
return next((i for i in group if i.mimetype.startswith('video/') and i.path.suffix != '.part'), None)
VIDEO_PROCESSING_LIMIT = 20
@register_modeler
def video_modeler(groups: Dict[str, List[File]], session: Session):
new_videos = []
async def video_modeler():
while True:
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup, Video).filter(
FileGroup.indexed != True,
FileGroup.mimetype.like('video/%'),
).outerjoin(Video, Video.file_group_id == FileGroup.id) \
.limit(VIDEO_PROCESSING_LIMIT)
file_groups: List[Tuple[FileGroup, Video]] = list(file_groups)
# Search all groups for video files.
video_files = {stem: video_file for stem, group in groups.items() if
(video_file := find_video_file_in_group(group))}
if not video_files:
# No videos in these groups.
return
# Get all matching Video records (if any) in one query.
video_paths = [i.path for i in video_files.values()]
video_records = {i.video_path: i for i in session.query(Video).filter(Video.video_path.in_(video_paths))}
processed = 0
for file_group, video in file_groups:
processed += 1
for stem, video_file in video_files.items():
group = groups[stem]
video_id = None
try:
if not video:
video = Video(file_group=file_group)
session.add(video)
session.flush([video])
video_id = video.id
video.validate()
except Exception as e:
if PYTEST:
raise
logger.error(f'Unable to model Video {video_id=}', exc_info=e)
session.flush(group)
poster_file = info_json_file = caption_file = None
for file in group:
if file.mimetype.startswith('image/'):
poster_file = file
elif file.path.name.endswith('.info.json'):
info_json_file = file
# Prefer WebVTT over SRT. (SRT cannot be displayed for HTML video).
elif file.path.name.endswith('.en.vtt'):
caption_file = file
elif file.path.name.endswith('.en.srt'):
caption_file = caption_file or file
file_group.indexed = True
if poster_file:
poster_file.associated = True
poster_file.do_stats()
if caption_file:
caption_file.associated = True
caption_file.do_stats()
if info_json_file:
info_json_file.associated = True
info_json_file.do_stats()
if processed < VIDEO_PROCESSING_LIMIT:
# Did not reach limit, do not query again.
break
video: Video = video_records.get(video_file.path)
if not video:
video = Video(video_file=video_file)
session.add(video)
size = video_file.path.stat().st_size
if poster_file != video.poster_file or \
caption_file != video.caption_file or \
info_json_file != video.info_json_file or \
video.size != size:
# Files might have been changed. Re-index.
video.video_file.indexed = False
video.poster_file = poster_file
video.caption_file = caption_file
video.info_json_file = info_json_file
video.size = size
video_file.model = Video.__tablename__
new_videos.append(video)
if new_videos:
session.flush(new_videos)
for video in new_videos:
video.video_file.do_index()
video.validate(session)
# Remove this group, it will not be processed by other modelers.
stem, _ = split_path_stem_and_suffix(video.video_path)
del groups[stem]
logger.debug(f'Modeled {processed} videos')
# Sleep to catch cancel.
await asyncio.sleep(0)
@register_after_refresh
@limit_concurrent(1)
def video_cleanup():
# Claim all Videos in a Channel's directory for that Channel. But, only if they have not yet been claimed.
logger.info('Claiming Videos for their Channels')
with get_db_curs(commit=True) as curs:
# Delete all Videos if the FileModel no longer contains a video.
curs.execute('''
WITH deleted AS
(UPDATE file_group SET model=null WHERE model='video' AND mimetype NOT LIKE 'video/%' RETURNING id)
DELETE FROM video WHERE file_group_id = ANY(select id from deleted)
''')
# Claim all Videos in a Channel's directory for that Channel. But, only if they have not yet been claimed.
curs.execute('''
UPDATE video v
SET channel_id = c.id
SET
channel_id = c.id
FROM channel c
LEFT JOIN file_group fg ON fg.primary_path LIKE c.directory || '/%'::VARCHAR
WHERE
v.video_path LIKE c.directory || '/%'::VARCHAR
AND v.channel_id IS NULL
v.channel_id IS NULL
AND fg.id = v.file_group_id
''')
curs.execute('''
UPDATE file_group fg
SET
-- Update channel information in file_group.data
-- Example: data = {'channel': {'id': 1, 'name': 'example'}}
data = fg.data::jsonb || jsonb_build_object('channel',
jsonb_build_object('id', v.channel_id, 'name', c.name))
FROM video v
LEFT JOIN channel c on c.id = v.channel_id
WHERE fg.id = v.file_group_id
''')
EXTRACT_SUBTITLES = False
@register_indexer('video')
class VideoIndexer(Indexer, ABC):
"""Handles video files like mp4/ogg."""
@staticmethod
def get_description(file):
video_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
info_json_path = video_path.with_suffix('.info.json')
if info_json_path.is_file():
with info_json_path.open('rt') as fh:
try:
return json.load(fh)['description']
except Exception:
if not PYTEST:
logger.warning(f'Video info json file exists, but cannot get description. {file}')
return None
@classmethod
def create_index(cls, file):
"""
Index the video file and it's associated files.
a = title.
b = <empty>
c = description from info json.
d = captions from info json, or extracted from video file.
"""
a = cls.get_title(file)
c = cls.get_description(file)
# Detect the associated caption files.
d = None
if (en_vtt := file.path.with_suffix('.en.vtt')).is_file():
d = read_captions(en_vtt)
elif (vtt := file.path.with_suffix('.vtt')).is_file():
d = read_captions(vtt)
elif (en_srt := file.path.with_suffix('.en.srt')).is_file():
d = read_captions(en_srt)
elif (srt := file.path.with_suffix('.srt')).is_file():
d = read_captions(srt)
elif EXTRACT_SUBTITLES:
d = extract_captions(file.path) or ''
d = split_lines_by_length(d)
return a, None, c, d

View file

@ -36,18 +36,16 @@ def download(_, channel_id: int = None):
return response.empty()
@content_bp.post('/favorite')
@content_bp.post('/tag')
@openapi.definition(
description='Toggle the favorite flag on a video',
body=schema.FavoriteRequest,
description='Associate a Video with a Tag',
body=schema.TagRequest,
)
@validate(schema.FavoriteRequest)
@openapi.response(HTTPStatus.OK, schema.FavoriteResponse)
@validate(schema.TagRequest)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
async def favorite(_: Request, body: schema.FavoriteRequest):
_favorite = video_lib.set_video_favorite(body.video_id, body.favorite)
ret = {'video_id': body.video_id, 'favorite': _favorite}
return json_response(ret, HTTPStatus.OK)
async def tag(_: Request, body: schema.TagRequest):
video_lib.tag_video(body.video_id, body.tag_name)
return response.empty()
@content_bp.get('/statistics')

View file

@ -58,8 +58,6 @@ async def get_channels_video_count() -> Dict[int, int]:
FROM
channel AS c
LEFT JOIN video AS v ON v.channel_id = c.id
WHERE
v.video_path IS NOT NULL
GROUP BY 1
'''
curs.execute(stmt)

View file

@ -28,10 +28,10 @@ def test_get_channels(test_directory, channel_factory, test_client):
def test_get_video(test_client, test_session, simple_channel, video_factory):
"""Test that you get can information about a video. Test that video file can be gotten."""
now_ = now()
video1 = video_factory(channel_id=simple_channel.id)
video1.upload_date, video1.title = now_, 'vid1'
video2 = video_factory(channel_id=simple_channel.id)
video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2'
video1 = video_factory(channel_id=simple_channel.id, title='vid1')
video1.upload_date = now_
video2 = video_factory(channel_id=simple_channel.id, title='vid2')
video2.upload_date = now_ + timedelta(seconds=1)
test_session.commit()
# Test that a 404 is returned when no video exists
@ -42,11 +42,11 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
# Get the video info we inserted
_, response = test_client.get('/api/videos/video/1')
assert response.status_code == HTTPStatus.OK, response.json
assert_dict_contains(response.json['file']['video'], {'title': 'vid1'})
assert_dict_contains(response.json['file_group'], {'title': 'vid1'})
# The next video is included.
assert response.json['prev'] is None
assert_dict_contains(response.json['next']['video'], {'title': 'vid2'})
assert_dict_contains(response.json['next'], {'title': 'vid2'})
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
@ -173,8 +173,8 @@ def test_video_file_name(test_session, simple_video, test_client):
"""If a Video has no title, the front-end can use the file name as the title."""
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
assert resp.status_code == HTTPStatus.OK
assert resp.json['file']['video']['video_path'] == 'simple_video.mp4'
assert resp.json['file']['video'].get('stem') == 'simple_video'
assert resp.json['file_group']['video']['video_path'] == 'simple_video.mp4'
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
def test_channel_conflicts(test_client, test_session, test_directory):
@ -443,7 +443,7 @@ def test_search_videos_channel(test_client, test_session, video_factory):
d = dict(channel_id=channel1.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['files']) == 0
assert len(response.json['file_groups']) == 0
with get_db_session(commit=True) as session:
vid1 = video_factory(title='vid1', channel_id=channel2.id)
@ -454,15 +454,15 @@ def test_search_videos_channel(test_client, test_session, video_factory):
# Videos are gotten by their respective channels
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['files']) == 1
assert response.json['totals']['files'] == 1
assert_dict_contains(response.json['files'][0], dict(path='vid2.mp4', video=dict(channel_id=channel1.id)))
assert len(response.json['file_groups']) == 1
assert response.json['totals']['file_groups'] == 1
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
d = dict(channel_id=channel2.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['files']) == 1
assert_dict_contains(response.json['files'][0], dict(path='vid1.mp4', video=dict(channel_id=channel2.id)))
assert len(response.json['file_groups']) == 1
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):
@ -488,7 +488,7 @@ def test_get_channel_videos_pagination(test_client, test_session, simple_channel
last_ids = []
for offset, video_count in tests:
_, response = assert_video_search(channel_id=simple_channel.id, order_by='id', offset=offset, limit=20)
assert len(response.json['files']) == video_count, 'Returned videos does not match'
current_ids = [i['video']['id'] for i in response.json['files']]
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
current_ids = [i['id'] for i in response.json['file_groups']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
last_ids = current_ids

View file

@ -1,3 +1,4 @@
import asyncio
from unittest import mock
import pytest
@ -92,40 +93,3 @@ def test_channels_no_url(test_session, test_directory):
save_channels_config()
import_channels_config()
@pytest.mark.asyncio
async def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory):
"""A Video will be assigned to a Channel by its info json first, then to its directory."""
channel1 = channel_factory(source_id='channel1')
channel2 = channel_factory(source_id='channel2')
# Put the video in the wrong directory. It should be matched to the info json channel first, then to it's directory
channel1.source_id = 'channel source id'
vid1 = video_factory(
with_video_file=(channel2.directory / 'video1.mp4'),
with_info_json={'channel_id': channel1.source_id},
)
test_session.commit()
assert not vid1.channel_id
with mock.patch('wrolpi.common.after_refresh') as mock_after_refresh:
# Channel can be found in `video_cleanup`, prevent that with mock.
mock_after_refresh.side_effect = []
await refresh_files()
# Channel in info_json was found during validation.
assert vid1.channel_id == channel1.id
# Refresh using cleanup, info json should be trusted.
await refresh_files()
assert vid1.channel_id == channel1.id
# Remove the info json, the Video's Channel will be its directory.
vid1.channel_id = None
vid1.info_json_path.unlink()
test_session.commit()
await refresh_files()
test_session.refresh(vid1)
vid1 = test_session.query(Video).one()
assert vid1.channel_id == channel2.id

View file

@ -12,35 +12,3 @@ def test_delete_channel_no_url(test_session, test_client, channel_factory):
test_session.commit()
channel.delete_with_videos()
@pytest.mark.asyncio
async def test_channel_refresh_censored(test_session, channel_factory, video_factory):
"""Censored videos are discovered when refreshing a channel's directory."""
channel = channel_factory()
channel.info_json = {
'entries': [
{'id': 'one', 'view_count': 100},
{'id': 'two', 'view_count': 200},
# Three was deleted, it should be marked as censored.
# {'id': 'three', 'view_count': 300},
]
}
vid1 = video_factory(channel_id=channel.id, source_id='one')
vid2 = video_factory(channel_id=channel.id, source_id='two')
vid3 = video_factory(channel_id=channel.id, source_id='three')
test_session.commit()
assert channel.refreshed is False
assert vid1.censored is False and vid1.view_count is None
assert vid2.censored is False and vid1.view_count is None
assert vid3.censored is False and vid1.view_count is None
await channel.refresh_files()
channel = test_session.query(Channel).one()
assert channel.refreshed is True
assert vid1.censored is False and vid1.view_count == 100
assert vid2.censored is False and vid2.view_count == 200
assert vid3.censored is True and vid3.view_count is None

View file

@ -6,9 +6,8 @@ import subprocess
import tempfile
from datetime import timedelta
from decimal import Decimal
from functools import partial
from pathlib import Path
from typing import List, Optional, Tuple
from typing import Optional, Tuple
import PIL
from PIL import Image
@ -16,25 +15,16 @@ from sqlalchemy.orm import Session
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import FFPROBE_BIN
from wrolpi.common import logger, iterify, get_media_directory, \
minimize_dict, match_paths_to_suffixes
from wrolpi.common import logger, get_media_directory
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import ChannelNameConflict, ChannelURLConflict, \
ChannelDirectoryConflict, ChannelSourceIdConflict
from wrolpi.files.models import File
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
from .models import Channel
logger = logger.getChild(__name__)
REQUIRED_OPTIONS = ['name', 'directory']
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count'}
MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'}
MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size',
'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id',
'view_count'}
# These are the supported video formats. These are in order of their preference.
VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv')
class ConfigError(Exception):
@ -145,7 +135,7 @@ def generate_video_poster(video_path: Path) -> Tuple[Path, Optional[int]]:
def convert_image(existing_path: Path, destination_path: Path, ext: str = 'jpeg'):
"""Convert an image from one format to another. Remove the existing image file. This will safely overwrite an
image if the existing path is the same as the destination path.
image if the destination file already exists.
"""
with tempfile.NamedTemporaryFile(dir=destination_path.parent, delete=False) as fh:
img = Image.open(existing_path).convert('RGB')
@ -225,11 +215,8 @@ def check_for_video_corruption(video_path: Path) -> bool:
return corrupt
def apply_info_json(channel_id: int):
"""Update view_count for all Videos in a channel using its info_json file.
Mark any videos not in the info_json as "censored".
"""
def update_view_counts(channel_id: int):
"""Update view_count for all Videos in a channel using its info_json file."""
with get_db_session() as session:
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
channel_name = channel.name
@ -255,47 +242,3 @@ def apply_info_json(channel_id: int):
curs.execute(stmt, (view_counts_str, channel_id))
count = len(curs.fetchall())
logger.debug(f'Updated {count} view counts in DB for {channel_name}.')
# Mark any video not in the info_json entries as censored.
source_ids = [i['id'] for i in info['entries']]
stmt = '''
UPDATE video SET censored=(source_id != ALL(%s))
WHERE channel_id=%s
'''
curs.execute(stmt, (source_ids, channel_id))
minimize_channel = partial(minimize_dict, keys=MINIMUM_CHANNEL_KEYS)
minimize_video_info_json = partial(minimize_dict, keys=MINIMUM_INFO_JSON_KEYS)
_minimize_video = partial(minimize_dict, keys=MINIMUM_VIDEO_KEYS)
def minimize_video(video: dict) -> dict:
"""
Return a Video dictionary that contains only the key/values typically used. Minimize the Channel and info_json,
if they are present.
"""
video = _minimize_video(video)
if video.get('channel'):
video['channel'] = minimize_channel(video['channel'])
if video.get('info_json'):
video['info_json'] = minimize_video_info_json(video['info_json'])
return video
match_video_paths = partial(match_paths_to_suffixes, suffix_groups=(
tuple(f'.{i}' for i in VIDEO_EXTENSIONS),
('.jpg', '.jpeg', '.webp', '.png'),
('.description',),
('.en.vtt', '.en.srt'),
('.info.json',),
))
@iterify(tuple)
def match_video_files(files: List[File]) -> Tuple[File, File, File, File, File]:
video_paths = match_video_paths([i.path for i in files])
for path in video_paths:
yield next((i for i in files if i.path == path), None)

View file

@ -14,7 +14,7 @@ from modules.videos.downloader import VideoDownloader, ChannelDownloader
from modules.videos.lib import set_test_channels_config, set_test_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
from wrolpi.files.models import File
from wrolpi.files.models import FileGroup
from wrolpi.vars import PROJECT_DIR
@ -71,13 +71,12 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
@pytest.fixture
def simple_video(test_session, test_directory, simple_channel) -> Video:
def simple_video(test_session, test_directory, simple_channel, video_file) -> Video:
"""A Video with an empty video file whose channel is the Simple Channel."""
video_path = test_directory / 'simple_video.mp4'
video_path.touch()
video_file = File(path=video_path, model='video')
video = Video(video_file=video_file, channel_id=simple_channel.id)
test_session.add(video)
video_file.rename(video_path)
video = Video.from_paths(test_session, video_path)
video.channel = simple_channel
test_session.commit()
video = test_session.query(Video).filter_by(id=video.id).one()
return video
@ -89,7 +88,7 @@ def video_factory(test_session, test_directory):
def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file=None,
with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False,
source_id: str = None):
source_id: str = None) -> Video:
title = title or str(uuid4())
if with_video_file and isinstance(with_video_file, (pathlib.Path, str)):
@ -104,13 +103,8 @@ def video_factory(test_session, test_directory):
(test_directory / 'videos/NO CHANNEL').mkdir(exist_ok=True, parents=True)
path = test_directory / f'videos/NO CHANNEL/{title}.mp4'
video_file = File(path=path, model='video')
test_session.add(video_file)
if with_video_file:
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
else:
path.touch()
# Create a real video file for mimetype.
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
info_json_path = None
if with_info_json:
@ -118,32 +112,25 @@ def video_factory(test_session, test_directory):
with_info_json = {'duration': 5} if with_info_json is True else with_info_json
info_json_path = path.with_suffix('.info.json')
info_json_path.write_text(json.dumps(with_info_json))
test_session.add(File(path=info_json_path))
poster_path = None
if with_poster_ext:
poster_path = path.with_suffix(f'.{with_poster_ext}')
poster_path = path.with_suffix(f'.{with_poster_ext.lstrip(".")}')
Image.new('RGB', (25, 25)).save(poster_path)
test_session.add(File(path=poster_path))
caption_path = None
if with_caption_file:
caption_path = path.with_suffix('.en.vtt')
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', caption_path)
test_session.add(File(path=caption_path))
video = Video(
video_path=path,
title=title,
channel_id=channel_id,
source_id=source_id or title,
info_json_path=info_json_path,
poster_path=poster_path,
caption_path=caption_path,
upload_date=upload_date,
)
test_session.add(video)
video_file.do_index()
paths = (path, info_json_path, poster_path, caption_path)
paths = list(filter(None, paths))
video = Video.from_paths(test_session, *paths)
video.channel_id = channel_id
video.source_id = source_id or title
video.upload_date = upload_date
video.validate()
return video
return factory
@ -167,9 +154,8 @@ def video_download_manager(test_download_manager) -> DownloadManager:
def test_channels_config(test_directory):
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/channels.yaml'
set_test_channels_config(True)
yield config_path
set_test_channels_config(False)
with set_test_channels_config():
yield config_path
@pytest.fixture
@ -213,14 +199,19 @@ def assert_video_ids(test_session):
@pytest.fixture
def video_with_search_factory(test_session, test_directory):
def video_with_search_factory(test_session, test_directory, video_file_factory):
"""A factory that creates a Video record with an associated video File record."""
def video_with_search(path: str = None, title: str = None, b_text: str = None, c_text: str = None,
d_text: str = None, mimetype: str = 'video/mp4'):
path = path or (test_directory / str(uuid4())).absolute()
video_file = File(path=path, a_text=title, d_text=d_text, mimetype=mimetype, model='video')
video = Video(title=title, video_file=video_file)
video_file_group = FileGroup.from_paths(test_session, video_file_factory(path))
video_file_group.a_text = title
video_file_group.b_text = b_text
video_file_group.c_text = c_text
video_file_group.d_text = d_text
video_file_group.mimetype = mimetype
video_file_group.model = 'video'
video = Video(file_group=video_file_group)
test_session.add(video)
return video
@ -238,7 +229,7 @@ def assert_video_search(test_client):
assert_ids: List[int] = None,
assert_paths: List[str] = None,
search_str: str = None,
filters: List[str] = None,
tag_names: List[str] = None,
offset: int = None,
limit: int = None,
order_by: str = None,
@ -247,8 +238,8 @@ def assert_video_search(test_client):
content = dict()
if search_str is not None:
content['search_str'] = search_str
if filters is not None:
content['filters'] = filters
if tag_names is not None:
content['tag_names'] = tag_names
if offset is not None:
content['offset'] = offset
if limit is not None:
@ -264,16 +255,15 @@ def assert_video_search(test_client):
assert response.status_code == HTTPStatus.OK
if assert_ids is not None:
assert 'files' in response.json, 'No video files in response'
assert [i['video']['id'] for i in response.json['files']] == assert_ids, 'Video IDs do not match'
assert 'file_groups' in response.json, 'No video file_groups in response'
assert [i['id'] for i in response.json['file_groups']] == assert_ids, 'Video IDs do not match'
if assert_paths is not None:
assert 'files' in response.json, 'No video files in response'
assert [i['video']['video_path'] for i in response.json['files']] == assert_paths, \
'Video paths do not match'
assert 'file_groups' in response.json, 'No video file_groups in response'
assert [i['primary_path'] for i in response.json['file_groups']] == assert_paths, 'Video paths do not match'
if assert_total:
assert response.json['totals']['files'] == int(assert_total), 'Total video files does not match'
if assert_total and not (count := response.json['totals']['file_groups']) == int(assert_total):
raise AssertionError(f'Video count does not match: {count} != {assert_total}')
return request, response

View file

@ -5,6 +5,7 @@ import re
import traceback
from abc import ABC
from typing import Tuple, List, Optional, Dict
from urllib.parse import urlparse
import yt_dlp.utils
from sqlalchemy.orm import Session
@ -15,18 +16,18 @@ from yt_dlp.utils import UnsupportedError, DownloadError
from wrolpi.cmd import YT_DLP_BIN
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.db import get_db_session
from wrolpi.db import optional_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.files.models import File
from wrolpi.files.lib import glob_shared_stem
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import apply_info_json, get_no_channel_directory, get_videos_directory
from .common import get_no_channel_directory, get_videos_directory, update_view_counts
from .lib import get_downloader_config
from .models import Video, Channel
from .schema import ChannelPostRequest
from .video_url_resolver import normalize_youtube_shorts_url
from .video_url_resolver import normalize_youtube_shorts_url, video_url_resolver
logger = logger.getChild(__name__)
ydl_logger = logger.getChild('youtube-dl')
@ -319,13 +320,11 @@ class VideoDownloader(Downloader, ABC):
)
with get_db_session(commit=True) as session:
video_file = File.upsert(video_path, session)
video: Video = Video.upsert(video_file, session)
# Find any files downloaded with the video (poster, caption, etc.).
video_paths = glob_shared_stem(video_path)
video = Video.from_paths(session, *video_paths)
video.source_id = entry['id']
video.channel_id = channel_id
video.video_file.do_index()
video.validate(session)
session.flush([video])
video_id = video.id
except UnrecoverableDownloadError:
raise
@ -464,9 +463,6 @@ def update_channel_catalog(channel: Channel, info: dict):
logger.warning(f'entries: {entries}')
raise KeyError('No id key for entry!') from e
# In order to store the Video's URL, we will need a quick lookup.
urls = {i['id']: i.get('webpage_url') for i in entries}
with get_db_session(commit=True) as session:
# Get the channel in this new context.
channel: Channel = session.query(Channel).filter_by(id=channel.id).one()
@ -474,20 +470,7 @@ def update_channel_catalog(channel: Channel, info: dict):
channel.info_json = info
channel.info_date = now()
channel.source_id = info.get('id')
with get_db_curs() as curs:
# Get all known videos in this channel.
query = 'SELECT source_id FROM video WHERE channel_id=%s AND source_id IS NOT NULL'
curs.execute(query, (channel.id,))
known_source_ids = {i[0] for i in curs.fetchall()}
new_source_ids = all_source_ids.difference(known_source_ids)
logger.info(f'Got {len(new_source_ids)} new videos for channel {channel.name}')
channel_id = channel.id
for source_id in new_source_ids:
url = urls.get(source_id)
session.add(Video(source_id=source_id, channel_id=channel_id, url=url))
# Write the Channel's info to a JSON file.
if channel.directory:
@ -496,30 +479,44 @@ def update_channel_catalog(channel: Channel, info: dict):
json.dump(info, fh, indent=2)
# Update all view counts using the latest from the Channel's info_json.
apply_info_json(channel_id)
update_view_counts(channel_id)
def _find_all_missing_videos(channel_id: id) -> List[Tuple]:
"""Get all Video entries which don't have the required media files (i.e. hasn't been downloaded)."""
with get_db_curs() as curs:
query = f'''
SELECT
video.id, video.source_id, video.channel_id
FROM
video
LEFT JOIN channel ON channel.id = video.channel_id
WHERE
channel.url IS NOT NULL
AND channel.url != ''
AND video.source_id IS NOT NULL
AND channel_id = %s
AND video.channel_id IS NOT NULL
AND (video_path IS NULL OR video_path = '' OR poster_path IS NULL OR poster_path = '')
'''
params = (channel_id,)
curs.execute(query, params)
missing_videos = list(curs.fetchall())
return missing_videos
with get_db_session() as session:
channel: Channel = Channel.get_by_id(channel_id)
if not channel:
raise UnknownChannel(f'Cannot find channel with id {channel_id}')
def resolve_url(entry: dict) -> str:
parsed = urlparse(entry.get('webpage_url') or entry['url'])
domain = parsed.hostname
url_ = video_url_resolver(domain, entry)
url_ = normalize_youtube_shorts_url(url_)
return url_
info_json = channel.info_json
# (source_id, url, channel_id)
channel_videos = [(i['id'], resolve_url(i), channel_id) for i in info_json['entries']]
video_source_ids = [i[0] for i in channel_videos]
videos_with_source_id = list(session.query(Video).filter(Video.source_id.in_(video_source_ids)))
found_source_ids = set()
missing_files = []
for video in videos_with_source_id:
found_source_ids |= {video.source_id, }
if not video.poster_path:
missing_files.append(video)
elif not video.video_path:
missing_files.append(video)
missing_source_ids = set(video_source_ids) - found_source_ids
for source_id in missing_source_ids:
yield None, source_id, channel_id
for video in missing_files:
yield video.id, video.source_id, channel_id
async def find_all_missing_videos(channel_id: int = None):

View file

@ -1,20 +1,24 @@
import contextlib
import datetime
import functools
import html
import pathlib
import re
from collections import defaultdict
from typing import Tuple, Optional, Union, Generator
import warnings
from typing import Tuple, Optional, Generator
from sqlalchemy import or_
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi import before_startup
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh
from wrolpi.dates import from_timestamp, Seconds
from wrolpi.captions import extract_captions
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh, limit_concurrent
from wrolpi.dates import Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.errors import UnknownDirectory
from wrolpi.files.models import File
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag
from wrolpi.vars import PYTEST
from .common import is_valid_poster, convert_image, \
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
@ -61,7 +65,10 @@ def get_channel_id_by_source_id(session: Session, source_id: str) -> Optional[in
return channel.id if channel else None
def validate_video(video: Video, channel_generate_poster: bool, session: Session):
EXTRACT_SUBTITLES = False
def validate_video(video: Video, channel_generate_poster: bool):
"""
Validate a single video.
@ -71,13 +78,13 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
A Video is also valid when it has a JPEG poster, if any. If no poster can be
found, it will be generated from the video file.
"""
info_json_path = video.info_json_file.path if video.info_json_file else video.info_json_path
json_data_missing = bool(video.title) and bool(video.duration) and bool(video.view_count) and bool(video.url) \
and bool(video.upload_date)
info_json_path = video.info_json_path
json_data_missing = bool(video.file_group.title) and bool(video.duration) and bool(video.view_count) \
and bool(video.url) and bool(video.upload_date)
if info_json_path and json_data_missing is False:
# These properties can be found in the info json.
video_info_json = process_video_info_json(video)
video.title = video_info_json.title
video.file_group.title = video_info_json.title
video.duration = video_info_json.duration
video.url = video_info_json.url
# View count will probably be overwritten by more recent data when this Video's Channel is
@ -89,35 +96,40 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
video_path = video.video_path
if not video.title or not video.upload_date or not video.source_id:
if video_path and (not video.file_group.title or not video.upload_date or not video.source_id):
# Video is missing things that can be extracted from the video file name.
# These are the least trusted, so anything already on the video should be trusted.
_, upload_date, source_id, title = parse_video_file_name(video_path)
video.title = video.title or html.unescape(title)
video.upload_date = video.upload_date or upload_date
upload_date = upload_date or video.upload_date
if upload_date:
upload_date = datetime.datetime.strptime(upload_date, '%Y%M%d')
upload_date = upload_date.replace(tzinfo=datetime.timezone.utc)
video.file_group.title = video.file_group.title or html.unescape(title)
video.upload_date = upload_date
video.source_id = video.source_id or source_id
if not video.size or not video.modification_datetime:
stat = video_path.stat()
video.size = video.size or stat.st_size
video.modification_datetime = video.modification_datetime or from_timestamp(stat.st_mtime)
if channel_generate_poster:
# Try to convert/generate, but keep the old poster if those fail.
new_poster_path, duration = convert_or_generate_poster(video) or video.poster_path
if new_poster_path:
# Poster was created/updated.
poster_file = File.upsert(new_poster_path, session)
session.flush([poster_file])
video.poster_file = poster_file
video.poster_path = new_poster_path
video.file_group.append_files(new_poster_path)
if duration:
video.duration = duration
if not video.duration:
if video_path and not video.duration:
# Duration was not retrieved during poster generation.
video.duration = get_video_duration(video_path)
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:
# Caption file was deleted, clear out old captions.
video.file_group.d_text = None
elif video_path and not video.file_group.d_text and EXTRACT_SUBTITLES:
# Captions were not found, extract them from the video.
video.file_group.d_text = extract_captions(video_path)
elif video_path and video.caption_paths and not video.file_group.d_text:
video.file_group.d_text = video.get_caption_text()
def convert_or_generate_poster(video: Video) -> Tuple[Optional[pathlib.Path], Optional[int]]:
"""
@ -128,28 +140,32 @@ def convert_or_generate_poster(video: Video) -> Tuple[Optional[pathlib.Path], Op
"""
video_path = video.video_path
# Modification/generation of poster is enabled for this channel.
if video.poster_path or video.poster_file:
if video.poster_path:
# Check that the poster is a more universally supported JPEG.
old: pathlib.Path = video.poster_path if video.poster_path else video.poster_file.path
old: pathlib.Path = video.poster_path
new = old.with_suffix('.jpg')
if old != new and new.exists():
# Destination JPEG already exists (it may have the wrong format).
old.unlink()
old = video.poster_path = new
old = new
if not is_valid_poster(old):
# Poster is not valid, convert it and place it in the new location.
try:
convert_image(old, new)
old.unlink(missing_ok=True)
if not new.is_file():
raise FileNotFoundError(f'Failed to convert poster: {new}')
if old != new:
# Only remove the old one if we are not converting in-place.
old.unlink(missing_ok=True)
logger.info(f'Converted invalid poster {repr(str(old))} to {repr(str(new))}')
return new, None
except Exception as e:
logger.error(f'Failed to convert invalid poster {old} to {new}', exc_info=e)
return None, None
if not video.poster_path and not video.poster_file:
if not video.poster_path:
# Video poster was not discovered, or converted. Let's generate it.
try:
poster_path, duration = generate_video_poster(video_path)
@ -185,10 +201,12 @@ class ChannelsConfig(ConfigFile):
@property
def favorites(self) -> dict:
warnings.warn('Favorites have bee moved to Tags', DeprecationWarning)
return self._config['favorites']
@favorites.setter
def favorites(self, value: dict):
warnings.warn('Favorites have bee moved to Tags', DeprecationWarning)
self.update({'favorites': value})
@ -205,12 +223,12 @@ def get_channels_config():
return CHANNELS_CONFIG
def set_test_channels_config(enable: bool):
@contextlib.contextmanager
def set_test_channels_config():
global TEST_CHANNELS_CONFIG
if enable is True:
TEST_CHANNELS_CONFIG = ChannelsConfig()
else:
TEST_CHANNELS_CONFIG = None
TEST_CHANNELS_CONFIG = ChannelsConfig()
yield
TEST_CHANNELS_CONFIG = None
class VideoDownloaderConfig(ConfigFile):
@ -334,46 +352,14 @@ def get_channels_config_from_db(session: Session) -> dict:
"""Create a dictionary that contains all the Channels from the DB."""
channels = session.query(Channel).order_by(Channel.directory).all()
channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory'])
# Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel
# is deleted or the DB is wiped.
favorite_videos = session.query(Video).filter(Video.favorite != None, Video.video_path != None).all() # noqa
favorites = defaultdict(lambda: {})
for video in favorite_videos:
if video.channel:
directory = video.channel.directory.relative_to(get_media_directory())
favorites[str(directory)][video.video_path.name] = dict(favorite=video.favorite)
else:
favorites['NO CHANNEL'][video.video_path.name] = dict(favorite=video.favorite)
favorites = dict(favorites)
return dict(channels=channels, favorites=favorites)
def _detect_old_favorites(config: Union[ChannelsConfig, dict]) -> bool:
# TODO remove these old favorites after beta.
media_directory = get_media_directory()
favorites = config.favorites if hasattr(config, 'favorites') else config['favorites']
for channel, favorites in favorites.items():
if channel == 'NO CHANNEL':
continue
if not (media_directory / channel).is_dir():
return True
return False
return dict(channels=channels)
@optional_session()
def save_channels_config(session=None, preserve_favorites: bool = True):
def save_channels_config(session: Session = None):
"""Get the Channel information from the DB, save it to the config."""
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
# TODO remove these old favorites after beta.
old_favorites = _detect_old_favorites(channels_config)
if old_favorites and config['favorites']:
# There are favorites in the database, remove the old favorites.
channels_config.favorites = {}
if preserve_favorites:
config['favorites'].update(channels_config.favorites or {})
channels_config.update(config)
@ -382,6 +368,7 @@ channel_import_logger = logger.getChild('channel_import')
@before_startup
@register_after_refresh
@limit_concurrent(1)
def import_channels_config():
"""Import channel settings to the DB. Existing channels will be updated."""
if PYTEST and not TEST_CHANNELS_CONFIG:
@ -469,15 +456,19 @@ def import_channels_config():
else:
channel_dir = get_no_channel_directory()
favorites_tag = Tag.find_by_name('Favorite')
if not favorites_tag:
channel_import_logger.warning(f'Cannot tag favorite videos without Favorite tag!')
return
# Set favorite Videos of this Channel.
for video_path, data in favorites.items():
# Favorite in the config is the name of the video_path. Add the channel directory onto this
# video_path, so we can match the complete path for the Video.
video_path = channel_dir / video_path
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
# If no Video is found, it may be that we need to refresh.
if video:
video.favorite = data['favorite']
file_group = FileGroup.find_by_path(video_path, session)
if file_group:
file_group.add_tag(favorites_tag, session)
else:
channel_import_logger.warning(f'Cannot find video to favorite: {video_path}')
except Exception as e:
@ -503,22 +494,12 @@ async def get_statistics():
SELECT
-- total videos
COUNT(id) AS "videos",
-- total videos that are marked as favorite
COUNT(id) FILTER (WHERE favorite IS NOT NULL) AS "favorites",
-- total videos downloaded over the past week/month/year
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 week') AS "week",
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 month') AS "month",
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 year') AS "year",
-- sum of all video lengths in seconds
COALESCE(SUM(duration), 0) AS "sum_duration",
-- sum of all video file sizes
COALESCE(SUM(size), 0)::BIGINT AS "sum_size",
-- largest video
COALESCE(MAX(size), 0) AS "max_size"
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 year') AS "year"
FROM
video
WHERE
video_path IS NOT NULL
''')
video_stats = dict(curs.fetchone())
@ -526,8 +507,7 @@ async def get_statistics():
curs.execute('''
SELECT
DATE_TRUNC('month', months.a),
COUNT(id)::BIGINT,
SUM(size)::BIGINT AS "size"
COUNT(id)::BIGINT
FROM
generate_series(
date_trunc('month', current_date) - interval '2 years',
@ -538,7 +518,6 @@ async def get_statistics():
video.upload_date >= date_trunc('month', months.a)
AND video.upload_date < date_trunc('month', months.a) + interval '1 month'
AND video.upload_date IS NOT NULL
AND video.video_path IS NOT NULL
GROUP BY
1
ORDER BY
@ -610,18 +589,11 @@ def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path
directory = str(directory).rstrip('/')
curs.execute(f'''
SELECT f1.path
FROM (
-- Get array of mimetypes for each full_stem group.
select f2.full_stem, array_agg(f2.mimetype) as mimetypes
from file f2
where f2.full_stem like '{directory}/%'
group by 1) AS stem_groups
-- Get all files that match the full_stem.
LEFT JOIN file f1 ON stem_groups.full_stem = f1.full_stem
-- Get all groups which DO NOT contain one of the video mimetypes.
WHERE NOT (select array_agg(f3.mimetype) from file f3 where f3.mimetype like 'video/%') && stem_groups.mimetypes
SELECT files
FROM file_group
WHERE
mimetype NOT LIKE 'video%'
AND full_stem LIKE '{directory}/%'
''')
results = map(lambda i: pathlib.Path(i[0]), curs.fetchall())
results = filter(lambda i: i.is_file(), results)
results = (pathlib.Path(j['path']) for i in curs.fetchall() for j in i[0])
yield from results

View file

@ -1,21 +1,23 @@
import datetime
import json
import pathlib
from datetime import datetime
from pathlib import Path
from typing import Optional, Generator, Dict, List
from typing import Optional, Dict, List, Union
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey
from sqlalchemy.orm import relationship, Session
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, BigInteger, any_
from sqlalchemy.orm import relationship, Session, deferred
from sqlalchemy.orm.collections import InstrumentedList
from wrolpi.captions import read_captions
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task
from wrolpi.dates import now, TZDateTime
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download, download_manager
from wrolpi.errors import UnknownVideo
from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import File
from wrolpi.files.lib import refresh_files, get_mimetype, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
from wrolpi.tags import Tag, TagFile
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@ -24,119 +26,73 @@ logger = logger.getChild(__name__)
class Video(ModelHelper, Base):
__tablename__ = 'video'
id = Column(Integer, primary_key=True)
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
validated = Column(Boolean, default=False)
censored = Column(Boolean, default=False)
duration = Column(Integer)
favorite = Column(TZDateTime)
modification_datetime = Column(TZDateTime)
size = Column(Integer)
source_id = Column(String)
title = Column(String)
upload_date = Column(TZDateTime)
upload_date: datetime.datetime = Column(TZDateTime)
url = Column(String)
view_count = Column(Integer)
viewed = Column(TZDateTime)
# Associated Files
caption_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
caption_file: File = relationship('File', primaryjoin='Video.caption_path==File.path')
info_json_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
info_json_file: File = relationship('File', primaryjoin='Video.info_json_path==File.path')
poster_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
poster_file: File = relationship('File', primaryjoin='Video.poster_path==File.path')
video_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
video_file: File = relationship('File', primaryjoin='Video.video_path==File.path')
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
file_group: FileGroup = relationship('FileGroup')
def __repr__(self):
v = self.video_file
v = None
if self.video_path:
video_path = self.video_path.path if hasattr(self.video_path, 'path') else self.video_path
v = video_path.relative_to(get_media_directory())
return f'<Video id={self.id} title={repr(self.title)} path={v} channel={self.channel_id} ' \
v = self.video_path.relative_to(get_media_directory())
return f'<Video id={self.id} title={repr(self.file_group.name)} path={v} channel={self.channel_id} ' \
f'source_id={repr(self.source_id)}>'
@classmethod
def upsert(cls, video_file: File, session: Session):
video = session.query(Video).filter_by(video_file=video_file).one_or_none()
if not video:
video = Video(video_file=video_file)
video: Video
video.video_path = video.video_path or video_file.path
video.video_file = video.video_file or video_file
video.size = video.video_path.stat().st_size
session.add(video)
def __json__(self) -> dict:
d = self.file_group.__json__()
video.find_my_files()
channel = None
if self.channel:
channel = dict(id=self.channel.id, name=self.channel.name)
# Claim the video file for this model.
video.video_file.model = cls.__tablename__
return video
def my_paths(self) -> Generator[Path, None, None]:
"""Return all paths of this Video. Returns nothing if all paths are None."""
if self.poster_path:
yield self.poster_path
if self.caption_path:
yield self.caption_path
if self.info_json_path:
yield self.info_json_path
if self.video_path:
yield self.video_path
def my_files(self) -> Generator[File, None, None]:
"""Return all Files of this Video. Returns nothing if all paths are None."""
if self.poster_file:
yield self.poster_file
if self.caption_file:
yield self.caption_file
if self.info_json_file:
yield self.info_json_file
if self.video_file:
yield self.video_file
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
d['video'] = dict(
caption=self.file_group.d_text,
caption_files=self.caption_files,
channel=channel,
channel_id=self.channel_id,
description=self.file_group.c_text or self.get_video_description(),
id=self.id,
info_json_file=self.info_json_file,
info_json_path=self.info_json_path,
poster_file=self.poster_file,
poster_path=self.poster_path,
source_id=self.source_id,
stem=split_path_stem_and_suffix(self.video_path)[0],
upload_date=self.upload_date.isoformat() if self.upload_date else None,
url=self.url,
video_path=self.video_path,
view_count=self.view_count,
)
return d
def delete(self):
"""
Remove all files and File records related to this video. Delete this Video record.
Add it to it's Channel's skip list.
"""
for path in self.my_paths():
path.unlink(missing_ok=True)
needs_save = bool(self.favorite)
"""Remove all files and File records related to this video. Delete this Video record.
Add it to it's Channel's skip list."""
self.add_to_skip_list()
self.favorite = None
session = Session.object_session(self)
session.commit()
session.delete(self)
for file in self.my_files():
session.delete(file)
if needs_save:
# Save the Channels config. This video was a favorite and that needs to be removed.
from modules.videos.lib import save_channels_config
save_channels_config(preserve_favorites=False)
self.file_group.delete()
def add_to_skip_list(self):
"""Add this video to it's Channel's skip list."""
if self.channel and self.source_id:
self.channel.add_video_to_skip_list(self.source_id)
def set_favorite(self, favorite: bool) -> Optional[datetime]:
self.favorite = now() if favorite else None
return self.favorite
def set_viewed(self):
self.viewed = now()
def get_info_json(self) -> Optional[Dict]:
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
info_json_path = self.info_json_file.path if self.info_json_file else self.info_json_path
info_json_path = self.info_json_path
if not info_json_path:
return
@ -145,10 +101,6 @@ class Video(ModelHelper, Base):
return json.load(fh)
except FileNotFoundError:
logger.warning(f'Unable to find info json file! {info_json_path}')
if self.video_path.is_file():
# Clear out the info_json only if the video file exists. We don't want to clear out the info_json if
# the drive hasn't been mounted.
self.info_json_file = self.info_json_path = None
return None
except Exception as e:
logger.warning(f'Unable to parse info json {self.info_json_path}', exc_info=e)
@ -165,11 +117,6 @@ class Video(ModelHelper, Base):
if description:
return description
if self.description_path:
with open(self.description_path, 'rt') as fh:
contents = fh.read()
return contents
def get_surrounding_videos(self):
"""
Get the previous and next videos around this video. The videos must be in the same channel.
@ -236,96 +183,111 @@ class Video(ModelHelper, Base):
return previous_video, next_video
def __json__(self):
from modules.videos.common import minimize_video_info_json
info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None
stem, _ = split_path_stem_and_suffix(self.video_path) if self.video_file else (None, None)
d = dict(
caption_path=self.caption_path,
censored=self.censored,
channel=self.channel,
channel_id=self.channel_id,
modification_datetime=self.modification_datetime,
duration=self.duration,
favorite=self.favorite,
id=self.id,
info_json=info_json,
info_json_path=self.info_json_path,
poster_path=self.poster_path,
size=self.size,
source_id=self.source_id,
stem=stem,
title=self.title,
upload_date=self.upload_date,
url=self.url,
validated=self.validated,
view_count=self.view_count,
video_path=self.video_path.relative_to(get_media_directory()),
viewed=self.viewed,
)
return d
def validate(self, session: Session):
"""Perform a validation of this video and it's files. Mark this video as validated if no errors occur."""
if not self.video_file:
def validate(self):
"""Perform a validation of this video and it's files."""
if not self.file_group.primary_path:
# Can't validate if there is no video file.
return False
logger.error(f'Unable to validate video {self.id} without primary file!')
from .lib import validate_video
try:
validate_video(self, self.channel.generate_posters if self.channel else False, session)
self.validated = True
validate_video(self, self.channel.generate_posters if self.channel else False)
except Exception as e:
logger.warning(f'Failed to validate video {self}', exc_info=e)
return self.validated
def find_my_files(self):
"""Search near the video file for it's associated files. (poster, caption, etc.)"""
from modules.videos.common import match_video_files
if not self.video_path:
raise ValueError("Can't find files when I don't have a video path!")
# Associated files share the same stem, but different extensions.
associated_paths = glob_shared_stem(self.video_path)
associated_paths.pop(associated_paths.index(self.video_path))
# Get the session from the video first, fallback to the video file.
session: Session = Session.object_session(self) or Session.object_session(self.video_file)
files = [File.upsert(i, session) for i in associated_paths]
_, poster_file, description_file, caption_file, info_json_file = match_video_files(files)
if poster_file:
poster_file.do_index()
poster_file.associated = True
if caption_file:
caption_file.do_index()
caption_file.associated = True
if info_json_file:
info_json_file.do_index()
info_json_file.associated = True
session.flush([i for i in files])
self.poster_file = poster_file
self.caption_file = caption_file
self.info_json_file = info_json_file
self.file_group.model = Video.__tablename__
self.file_group.a_text = self.file_group.title
self.file_group.c_text = self.get_video_description()
# self.file_group.d_text is handled in `validate_video`.
@staticmethod
def find_by_path(path, session):
video = session.query(Video).filter_by(video_path=path).one_or_none()
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
str_paths = list(map(str, paths))
file_group = session.query(FileGroup).filter(FileGroup.primary_path == any_(str_paths)).one_or_none()
if not file_group:
# This video has not been downloaded previously.
file_group = FileGroup.from_paths(session, *paths)
# Video may have been downloaded previously.
video = session.query(Video).filter(Video.file_group_id == file_group.id).one_or_none()
if not video:
video = Video(file_group=file_group)
video.validate()
session.add(video)
session.flush([video, file_group])
return video
@property
def info_json_file(self) -> Optional[dict]:
for file in self.file_group.my_json_files():
return file
@property
def info_json_path(self) -> Optional[pathlib.Path]:
if info_json_file := self.info_json_file:
return info_json_file['path']
@property
def video_path(self) -> Optional[pathlib.Path]:
if self.file_group.primary_path and get_mimetype(self.file_group.primary_path).startswith('video/'):
return self.file_group.primary_path
# No primary file somehow, return the first video file.
for file_group in self.file_group.my_video_files():
return file_group['path']
@property
def poster_file(self) -> Optional[dict]:
for file_group in self.file_group.my_poster_files():
return file_group
@property
def poster_path(self) -> Optional[pathlib.Path]:
if poster_file := self.poster_file:
return poster_file['path']
@property
def caption_files(self) -> List[dict]:
return self.file_group.my_files('text/srt') + self.file_group.my_files('text/vtt')
@property
def caption_paths(self) -> List[pathlib.Path]:
return [i['path'] for i in self.caption_files]
def get_caption_text(self) -> Optional[str]:
"""Search the FileGroup's files for a caption file. Return the captions from only the best caption file."""
caption_paths = self.caption_paths
# Some SRT files are more supported than others, these are their preferred order.
caption_text = None
if english_srt := [i for i in caption_paths if i.name.endswith('.en.srt')]:
caption_text = read_captions(english_srt[0])
elif srt := [i for i in caption_paths if i.name.endswith('.srt')]:
caption_text = read_captions(srt[0])
elif english_vtt := [i for i in caption_paths if i.name.endswith('.en.vtt')]:
caption_text = read_captions(english_vtt[0])
elif vtt := [i for i in caption_paths if i.name.endswith('.vtt')]:
caption_text = read_captions(vtt[0])
return caption_text
@staticmethod
@optional_session
def find_by_path(path, session: Session) -> Optional['Video']:
video = session.query(Video) \
.join(FileGroup, FileGroup.id == Video.file_group_id) \
.filter(FileGroup.primary_path == path).one_or_none()
return video
@staticmethod
def find_by_paths(paths: List[pathlib.Path], session) -> List:
videos = list(session.query(Video).filter(Video.video_path.in_(paths)))
return videos
@optional_session
def find_by_id(id_: int, session: Session) -> Optional['Video']:
video = session.query(Video).filter(Video.id == id_).one_or_none()
return video
@property
def primary_path(self):
return self.video_path
def add_tag(self, tag_or_tag_name: Union[Tag, str]) -> TagFile:
tag = Tag.find_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name
return self.file_group.add_tag(tag)
class Channel(ModelHelper, Base):
@ -342,7 +304,7 @@ class Channel(ModelHelper, Base):
source_id = Column(String)
refreshed = Column(Boolean, default=False)
info_json = Column(JSON)
info_json = deferred(Column(JSON))
info_date = Column(Date)
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
@ -414,7 +376,6 @@ class Channel(ModelHelper, Base):
calculate_duration=self.calculate_duration,
directory=str(self.directory),
download_frequency=self.download_frequency,
favorites={},
generate_posters=self.generate_posters,
match_regex=self.match_regex or '',
name=self.name,
@ -464,9 +425,10 @@ class Channel(ModelHelper, Base):
SELECT
SUM(size),
MAX(size),
COUNT(id)
COUNT(video.id)
FROM video
WHERE channel_id = %(id)s AND video_path IS NOT NULL
LEFT JOIN file_group fg on fg.id = video.file_group_id
WHERE channel_id = %(id)s
'''
curs.execute(stmt, dict(id=self.id))
size, largest_video, video_count = curs.fetchone()
@ -484,20 +446,26 @@ class Channel(ModelHelper, Base):
self_id = self.id
# Refresh all files within this channel's directory first.
await refresh_directory_files_recursively(self.directory, send_events=send_events)
await refresh_files(self.directory, send_events=send_events)
# Apply any info_json (discover censored videos, etc.) second.
from modules.videos.common import apply_info_json
# Apply any info_json (update view counts) second.
from modules.videos.common import update_view_counts
if PYTEST:
apply_info_json(self_id)
update_view_counts(self_id)
self.refreshed = True
else:
# Perform info_json in background task. Channel will be marked as refreshed after this completes.
async def _():
apply_info_json(self_id)
update_view_counts(self_id)
with get_db_session(commit=True) as session:
channel: Channel = session.query(Channel).filter(Channel.id == self_id).one()
channel.refreshed = True
background_task(_())
@staticmethod
@optional_session
def get_by_id(id_: int, session: Session = None) -> Optional['Channel']:
channel = session.query(Channel).filter_by(id=id_).one_or_none()
return channel

View file

@ -139,11 +139,12 @@ class VideoResponse:
@dataclass
class VideoSearchRequest:
search_str: Optional[str] = None
filters: List[str] = field(default_factory=list)
tags: List[str] = field(default_factory=list)
offset: Optional[int] = None
limit: Optional[int] = VIDEO_QUERY_LIMIT
order_by: Optional[str] = DEFAULT_VIDEO_ORDER
channel_id: Optional[int] = None
tag_names: List[str] = field(default_factory=list)
@dataclass
@ -158,15 +159,9 @@ class PaginationQuery:
@dataclass
class FavoriteRequest:
class TagRequest:
video_id: int
favorite: bool
@dataclass
class FavoriteResponse:
video_id: int
favorite: datetime
tag_name: str
@dataclass
@ -177,7 +172,6 @@ class WROLModeRequest:
@dataclass
class VideoStatistics:
videos: int
favorites: int
week: int
month: int
year: int

View file

@ -1,5 +1,4 @@
import json
import shutil
from http import HTTPStatus
from pathlib import Path
@ -7,101 +6,106 @@ import pytest
from modules.videos.models import Video, Channel
from modules.videos.video import lib as video_lib
from wrolpi.db import get_db_curs
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import File
from wrolpi.vars import PROJECT_DIR
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_refresh_videos_index(test_session, test_directory, video_factory):
"""A video discovered during refresh has files associated with it. Only the video file is indexed."""
"""The video modeler indexes video data into the Video's FileGroup."""
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
test_session.commit()
test_session.query(File).delete()
test_session.query(FileGroup).delete()
test_session.commit()
for _ in range(1):
# Refreshing twice does not change the results.
await refresh_files()
assert test_session.query(File).count() == 4
assert test_session.query(FileGroup).count() == 1
video: Video = test_session.query(Video).one()
assert video.video_file.indexed is True, 'Video was not indexed'
assert video.video_file.a_text, 'Video title was not indexed'
assert video.video_file.d_text, 'Video captions were not indexed'
assert not video.poster_file.indexed
assert not video.caption_file.indexed
assert not video.info_json_file.indexed
assert video.file_group.indexed is True, 'Video was not indexed'
assert video.file_group.a_text and video.file_group.title, 'Video title was not indexed'
assert video.file_group.d_text, 'Video captions were not indexed'
# Delete caption file, the video should be indexed again.
video: Video = test_session.query(Video).one()
video.caption_path.unlink()
for path in video.caption_paths:
path.unlink()
await refresh_files()
assert video.video_file.indexed is True, 'Video was not indexed'
assert video.video_file.a_text, 'Video title was not indexed'
assert not video.video_file.d_text, 'Video captions were not removed'
assert video.file_group.indexed is True, 'Video was not indexed'
assert video.file_group.a_text and video.file_group.title, 'Video title was not indexed'
assert not video.file_group.d_text, 'Video captions were not removed'
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory, video_file_factory,
image_bytes_factory):
subdir = test_directory / 'subdir'
subdir.mkdir()
# video1 is in a subdirectory, move its files into the subdirectory.
video1 = video_factory(channel_id=simple_channel.id, with_video_file=subdir / 'video1.mp4',
with_info_json=True,
with_poster_ext='jpg')
with_info_json=True, with_poster_ext='jpg')
test_session.commit()
test_session.delete(video1.poster_file)
video1.poster_path = None
video1.file_group.size = video1.file_group.data = None
video1.file_group.files = list()
# video2 is in the test directory.
video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
with_poster_ext='jpg')
video2.poster_path = video2.poster_file = video1.poster_path = None
video2.file_group.data = None
video2.file_group.files = []
test_session.commit()
assert not video1.size, 'video1 should not have size during creation'
assert not video1.file_group.size, 'video1 should not have size during creation'
# Create a video not in the DB.
vid3 = test_directory / 'vid3.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3)
vid3 = video_file_factory(test_directory / 'vid3.mp4')
# Orphan poster should be ignored.
orphan_poster = Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg')
orphan_poster.touch()
orphan_poster.write_bytes(image_bytes_factory())
# Create a bogus video in the channel.
video_file = File(path=test_directory / 'foo')
video_file.idempotency = None # File was not added in the current refresh.
bogus = Video(video_file=video_file, channel_id=simple_channel.id)
test_session.add(bogus)
test_session.commit()
video_file = test_directory / 'foo.mp4'
with get_db_curs(commit=True) as curs:
full_stem, _ = split_path_stem_and_suffix(video_file, full=True)
stmt = "INSERT INTO file_group (mimetype, full_stem, primary_path, indexed, files, model)" \
" values ('video/mp4', %(full_stem)s, %(primary_path)s, true, %(files)s, 'video') RETURNING id"
params = {'full_stem': full_stem, 'primary_path': str(video_file), 'files': list()}
curs.execute(stmt, params)
video4_id = curs.fetchall()[0][0]
stmt = "INSERT INTO video (file_group_id) values (%(video_id)s)"
curs.execute(stmt, {'video_id': str(video4_id)})
test_client.post('/api/files/refresh')
test_session.expire_all()
# Posters were found during refresh.
assert video1.poster_path
assert 'subdir' in str(video1.poster_path)
assert video2.poster_path
assert video1.file_group.my_poster_files()
assert 'subdir' in str(video1.file_group.my_poster_files()[0])
assert video2.file_group.my_poster_files()
# Missing video3 was found
video3: Video = test_session.query(Video).filter_by(id=4).one()
assert video3.video_path == vid3
assert len(set(test_session.query(Video.video_path))) == 3, 'Should have 3 distinct video files'
assert len(set(test_session.query(FileGroup.primary_path).filter(FileGroup.model == 'video'))) == 3, \
'Should have 3 distinct video files'
# Bogus video was removed.
assert not any('foo' in str(i.video_path) for i in test_session.query(Video).all())
assert not any('foo.mp4' in str(i.video_path) for i in test_session.query(Video).all())
# Orphan file was not deleted.
assert orphan_poster.is_file(), 'Orphan poster was removed!'
assert video1.size, 'video1 size was not found'
assert video1.file_group.size, 'video1 size was not found'
# Remove video1's poster, video1 should be updated.
video1_video_path = str(video1.video_path)
video1.poster_path.unlink()
test_client.post('/api/files/refresh')
video1 = test_session.query(Video).filter_by(video_path=video1_video_path).one()
video1 = Video.find_by_path(video1_video_path, test_session)
assert not video1.poster_path
@ -133,9 +137,9 @@ def test_channels_with_videos(test_session, test_client, test_directory, channel
assert test_session.query(Video).count() == 3, 'Did not find correct amount of video files.'
assert {i[0] for i in test_session.query(Channel.name)} == {'channel1', 'channel2'}, 'Channels were changed.'
vid1: Video = test_session.query(Video).filter_by(video_path=vid1_path).one()
vid2: Video = test_session.query(Video).filter_by(video_path=vid2_path).one()
vid3: Video = test_session.query(Video).filter_by(video_path=vid3_path).one()
vid1 = Video.find_by_path(vid1_path)
vid2 = Video.find_by_path(vid2_path)
vid3 = Video.find_by_path(vid3_path)
assert vid1.channel == channel1
assert vid2.channel == channel2
@ -156,7 +160,7 @@ def test_api_download_channel(test_session, test_client, simple_channel):
assert response.status_code == HTTPStatus.NO_CONTENT
def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory):
def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, assert_video_search):
"""Test that videos can be searched and that their order is by their textsearch rank."""
# These captions have repeated letters, so they will be higher in the ranking.
videos = [
@ -170,58 +174,56 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
video_with_search_factory(title=title, d_text=caption)
test_session.commit()
def assert_search(search_str, expected, limit=20, total=None):
d = json.dumps({'search_str': search_str, 'limit': limit})
_, resp = test_client.post('/api/videos/search', content=d)
assert resp.status_code == HTTPStatus.OK
response_ids = [i['video']['id'] for i in resp.json['files']]
assert response_ids == expected
assert resp.json['totals']['files'] == (total or len(expected))
# Repeated runs should return the same result
for _ in range(2):
# Only videos with a b are returned, ordered by the amount of b's
assert_search('b', [1, 2, 3, 4])
assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
# Only two captions have e
assert_search('e', [4, 1])
assert_video_search(search_str='e', assert_ids=[4, 1])
# Only 2 contains 2
assert_video_search(search_str='2', assert_ids=[2, ])
# Only two captions have d
assert_search('d', [1, 2])
assert_video_search(search_str='d', assert_ids=[1, 2])
# 5 can be gotten by it's title
assert_search('5', [5])
assert_video_search(search_str='5', assert_ids=[5, ])
# only video 1 has e and d
assert_search('e d', [1])
assert_video_search(search_str='e d', assert_ids=[1, ])
# video 1 and 4 have b and e, but 1 has more
assert_search('b e', [1, 4])
assert_video_search(search_str='b e', assert_ids=[1, 4])
# Check totals are correct even with a limit
assert_search('b', [1, 2], limit=2, total=4)
assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel):
def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel, tag_factory):
"""Search the Video table. This does not need to use a join with the File table."""
vid1 = video_factory(upload_date='2022-09-16', with_video_file=True)
vid2 = video_factory(upload_date='2022-09-17', with_video_file=True, channel_id=simple_channel.id)
vid3 = video_factory(upload_date='2022-09-18', with_video_file=True)
vid1.set_favorite(True)
vid1 = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
vid2 = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2', channel_id=simple_channel.id)
vid3 = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
tag1, tag2 = tag_factory(), tag_factory()
vid1.add_tag(tag1)
vid3.add_tag(tag2)
test_session.commit()
assert test_session.query(Video).count() == 3
assert test_session.query(File).count() == 3
assert test_session.query(FileGroup).count() == 3
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='upload_date')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-upload_date')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-upload_date', limit=2)
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-upload_date', limit=2, offset=1)
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-upload_date', channel_id=simple_channel.id)
assert_video_search(assert_total=1, assert_ids=[vid1.id], filters=['favorite'])
assert_video_search(assert_total=1, assert_ids=[vid1.id], filters=['favorite'], limit=1)
assert_video_search(assert_total=1, assert_ids=[vid1.id], tag_names=[tag1.name])
assert_video_search(assert_total=1, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
assert_video_search(assert_total=2, assert_ids=[vid3.id, vid1.id], tag_names=[tag1.name, tag2.name])
# No results, no total is returned from the DB.
assert_video_search(assert_total=0, assert_ids=[], filters=['favorite'], offset=1)
assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=1)
# Check all order_by.
for order_by in video_lib.VIDEO_ORDERS:

View file

@ -15,7 +15,7 @@ from wrolpi.dates import now
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import convert_image, apply_info_json, get_video_duration, generate_video_poster, is_valid_poster
from ..common import convert_image, update_view_counts, get_video_duration, generate_video_poster, is_valid_poster
from ..lib import save_channels_config, get_channels_config, import_channels_config
@ -101,22 +101,22 @@ def test_update_view_count(test_session, channel_factory, video_factory):
check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None})
# Channel 1 is updated, the other channels are left alone.
apply_info_json(channel1.id)
update_view_counts(channel1.id)
check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None})
# Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored.
apply_info_json(channel2.id)
update_view_counts(channel2.id)
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None})
# All videos are updated.
apply_info_json(channel3.id)
update_view_counts(channel3.id)
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
# An outdated view count will be overwritten.
vid = test_session.query(Video).filter_by(id=1).one()
vid.view_count = 8
check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14})
apply_info_json(channel1.id)
update_view_counts(channel1.id)
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
@ -130,128 +130,6 @@ def test_generate_video_poster(video_file):
assert poster_path.stat().st_size > 0
def test_update_censored_videos(test_session, video_factory, simple_channel):
vid1 = video_factory(channel_id=simple_channel.id)
vid2 = video_factory(channel_id=simple_channel.id)
vid3 = video_factory(channel_id=simple_channel.id)
vid4 = video_factory() # should not be censored because it has no channel.
test_session.commit()
def check_censored(expected):
for video_id, censored in expected:
video = test_session.query(Video).filter_by(id=video_id).one()
assert video.censored == censored
# Videos are not censored by default.
test_session.commit()
apply_info_json(simple_channel.id)
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)])
# All videos are in the info_json.
simple_channel.info_json = {
'entries': [
dict(id=vid1.source_id, view_count=0),
dict(id=vid2.source_id, view_count=0),
dict(id=vid3.source_id, view_count=0),
]
}
test_session.commit()
apply_info_json(simple_channel.id)
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)])
simple_channel.info_json = {
'entries': [
dict(id=vid1.source_id, view_count=0), # vid2 is missing
dict(id=vid3.source_id, view_count=0),
]
}
test_session.commit()
apply_info_json(simple_channel.id)
check_censored([(vid1.id, False), (vid2.id, True), (vid3.id, False), (vid4.id, False)])
simple_channel.info_json = {
'entries': [
dict(id=vid1.source_id, view_count=0), # vid2 is back, vid3 is missing.
dict(id=vid2.source_id, view_count=0),
]
}
test_session.commit()
apply_info_json(simple_channel.id)
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, True), (vid4.id, False)])
simple_channel.info_json = {
'entries': [] # all videos gone
}
test_session.commit()
apply_info_json(simple_channel.id)
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
# Channels without info json preserve their last censored.
simple_channel.info_json = None
test_session.commit()
apply_info_json(simple_channel.id)
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
def test_import_favorites(test_session, test_directory, simple_channel, video_factory, test_channels_config):
"""
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
favorited again.
A Video's favorited status can only be cleared by calling `Video.delete`.
"""
video_factory(channel_id=simple_channel.id) # never favorited
vid2 = video_factory(channel_id=None) # has no channel
vid3 = video_factory(channel_id=simple_channel.id)
favorite = vid3.favorite = vid2.favorite = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
test_session.commit()
vid2_video_path = vid2.video_path
favorites = {
str(simple_channel.directory.relative_to(test_directory)): {vid3.video_path.name: {'favorite': favorite}},
'NO CHANNEL': {vid2.video_path.name: {'favorite': favorite}},
}
# Save config, verify that favorite is set.
save_channels_config()
config = get_channels_config()
assert config.favorites == favorites
def import_and_verify(favorited_ids: List[int]):
with mock.patch('modules.videos.lib.get_channel_source_id', lambda i: 'foo'):
import_channels_config()
for video in test_session.query(Video).all():
if video.id in favorited_ids:
assert video.favorite
else:
assert not video.favorite
# Clear the favorite (as if the DB was wiped), verify that the favorite is imported and set.
vid2.favorite = None
import_and_verify([vid2.id, vid3.id])
# Removing the video does not delete the favorite. If the DB is wiped, we do not want to lose our favorites!
test_session.query(Video).filter_by(id=vid2.id).delete()
test_session.commit()
save_channels_config()
config = get_channels_config()
assert config.favorites == favorites
import_and_verify([vid3.id])
# Add vid2 again, it should be favorited on import.
vid2 = Video(video_path=vid2_video_path)
test_session.add(vid2)
test_session.commit()
import_and_verify([vid3.id, vid2.id])
# Deleting the Video in the model really removes the favorite status.
vid3.delete()
config = get_channels_config()
assert config.favorites == {'NO CHANNEL': {vid2.video_path.name: {'favorite': favorite}}}
import_and_verify([vid2.id])
def test_import_channel_downloads(test_session, channel_factory, test_channels_config):
"""Importing the Channels' config should create any missing download records"""
channel1 = channel_factory(source_id='foo')
@ -305,67 +183,6 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
assert next_download == str(download.next_download)
def test_import_channels_config_outdated(test_session, test_directory, channel_factory, test_channels_config,
video_factory):
"""The Channels' config used to use a "link" to differentiate channels, test that old configs can be imported."""
channel1 = channel_factory(name='Channel1', download_frequency=DownloadFrequency.weekly)
channel2 = channel_factory(name='Channel2', download_frequency=DownloadFrequency.weekly)
test_session.commit()
vid1 = video_factory(title='vid1', channel_id=channel1.id)
vid2 = video_factory(title='vid2', channel_id=channel2.id)
vid3 = video_factory(title='vid3')
vid4 = video_factory(title='vid4')
test_session.commit()
channel1_name, channel_1_directory = channel1.name, channel1.directory
channel2_name, channel_2_directory = channel2.name, channel2.directory
assert channel1.directory.exists() and channel1.directory.is_absolute()
assert channel2.directory.exists() and channel2.directory.is_absolute()
# Channel's used to have a relative directory in the DB.
test_session.execute('UPDATE channel SET directory=:directory WHERE id=2',
dict(directory=str(channel_2_directory.relative_to(test_directory))),
)
save_channels_config()
# The config has a list of Channels, use the old method of a "dict" with the sanitized name as the key.
channels_config = get_channels_config()
channels_config.channels = {sanitize_link(i['name']): i for i in channels_config.channels}
# Change frequency to verify that the channels are updated.
for link, channel in channels_config.channels.items():
channel['download_frequency'] = DownloadFrequency.biweekly
# Change the favorites to the old "link" method as well.
channels_config.favorites = {
sanitize_link(channel1.name): {str(vid1.video_path): {'favorite': now()}},
sanitize_link(channel2.name): {str(vid2.video_path): {'favorite': now()}},
'NO CHANNEL': {str(vid3.video_path): {'favorite': now()}},
}
channels_config.save()
assert not vid1.favorite and not vid2.favorite and not vid3.favorite and not vid3.favorite
assert test_session.query(Channel).count() == 2
with mock.patch('modules.videos.lib.get_channel_source_id') as mock_get_channel_source_id:
mock_get_channel_source_id.return_value = 'some source id'
import_channels_config()
# No new channels were created. Existing channels were updated.
assert test_session.query(Channel).count() == 2
channel1, channel2 = test_session.query(Channel).order_by(Channel.id)
assert channel1.name == channel1_name and str(channel1.directory) == str(channel_1_directory)
# Channel2's directory is now absolute.
assert channel2.name == channel2_name and str(channel2.directory) == str(channel_2_directory)
assert channel1.download_frequency == DownloadFrequency.biweekly
assert channel2.download_frequency == DownloadFrequency.biweekly
# Videos were favorited by their channel "link".
assert vid1.favorite and vid2.favorite and vid3.favorite
assert not vid4.favorite
def test_check_for_video_corruption(video_file, test_directory):
# The test video is not corrupt.
assert common.check_for_video_corruption(video_file) is False

View file

@ -13,27 +13,15 @@ from modules.videos.models import Channel, Video
from wrolpi.db import get_db_context
from wrolpi.downloader import Download, DownloadResult
from wrolpi.errors import InvalidDownload
from wrolpi.files.models import File
from wrolpi.test.common import skip_circleci
from wrolpi.test.test_downloader import HTTPDownloader
from wrolpi.vars import PROJECT_DIR
@pytest.mark.asyncio
async def test_find_all_missing_videos(test_session, channel_factory, video_factory):
async def test_find_all_missing_videos(test_session, channel_factory, video_factory, image_file):
channel1 = channel_factory(url='some url')
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]}
test_session.commit()
# Two videos are already downloaded.
video_factory()
video_factory()
# No missing videos
assert [] == [i async for i in find_all_missing_videos(channel1.id)]
# Create a video that has no video file.
video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo')
test_session.add(video)
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0, 'webpage_url': 'https://example.com'}]}
test_session.commit()
missing = [i async for i in find_all_missing_videos(channel1.id)]
@ -41,11 +29,25 @@ async def test_find_all_missing_videos(test_session, channel_factory, video_fact
# The video returned is the one we faked.
id_, source_id, entry = missing[0]
# Two videos were created for this test already.
assert id_ == 3
# No Video.id because the video is missing.
assert id_ is None
# The fake entry we added is regurgitated back.
assert entry == channel1.info_json['entries'][0]
# Create a video without a poster, attempt to download it.
video = video_factory(channel_id=channel1.id, with_video_file=True)
channel1.info_json = {'entries': [
{'id': 'foo', 'view_count': 0, 'webpage_url': 'https://example.com'},
{'id': video.source_id, 'view_count': 0, 'webpage_url': 'https://example.com/new'},
]}
test_session.commit()
missing = [i async for i in find_all_missing_videos(channel1.id)]
assert len(missing) == 2
_, new = missing
assert new == (video.id, video.source_id, channel1.info_json['entries'][1])
example_video_json = {
'age_limit': 0,
@ -174,11 +176,12 @@ async def test_download_no_channel(test_session, video_download_manager, video_f
assert str(video.video_path) == f'{channel_dir}/video.mp4'
@skip_circleci
@pytest.mark.asyncio
async def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
mock_video_extract_info, mock_video_prepare_filename,
mock_video_process_runner):
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos.
"""Downloading (updating the catalog of) a Channel updates it's info_json.
If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
@ -407,10 +410,8 @@ async def test_video_download(test_session, test_directory, simple_channel, vide
assert test_session.query(Channel).one()
video: Video = test_session.query(Video).one()
assert video.video_path.is_absolute(), 'Video path is not absolute'
assert video.poster_path == poster_path, 'Video poster was not discovered'
assert video.validated, 'Video was not validated'
assert video.video_file and video.video_file.indexed, 'Video was not indexed'
assert video.video_path.is_absolute() and video.video_path == video_path, 'Video path is not absolute'
assert video.video_path.is_absolute() and video.poster_path == poster_path, 'Video poster was not discovered'
@pytest.mark.asyncio
@ -433,11 +434,10 @@ async def test_download_result(test_session, test_directory, video_download_mana
assert download.url == 'https://example.com'
assert download.location == '/videos/channel/1/video/1'
# Video files are associated.
# Video has its files.
video = test_session.query(Video).one()
assert not video.video_file.associated, 'Video file should not be associated.'
assert video.video_path.is_file(), 'Video file was not found.'
assert video.poster_path.is_file(), 'Poster file was not found.'
assert video.poster_file.associated, 'Poster file was not associated.'
@pytest.mark.asyncio
@ -527,15 +527,16 @@ example_playlist_json = {
@pytest.mark.asyncio
async def test_download_playlist(test_session, test_directory, mock_video_extract_info, video_download_manager):
async def test_download_playlist(test_session, test_directory, mock_video_extract_info, video_download_manager,
video_file):
"""All videos in a playlist can be downloaded for it's Channel."""
download = Download(url='playlist url')
test_session.add(download)
channel = get_or_create_channel(example_playlist_json['channel_id'], download.url, example_channel_json['uploader'])
video_file = File(path=test_directory / 'video file.mp4')
test_session.add(video_file)
test_session.add(Video(url='https://www.youtube.com/watch?v=video_1_url',
source_id='video 1 id', channel_id=channel.id, video_file=video_file))
video = Video.from_paths(test_session, video_file)
video.channel_id = channel.id
video.source_id = 'video 1 id'
video.url = 'https://www.youtube.com/watch?v=video_1_url'
test_session.commit()
mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first.

View file

@ -1,55 +1,19 @@
import datetime
import pathlib
import shutil
import pytest
import sqlalchemy
from modules.videos import lib
from modules.videos.common import apply_info_json
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
from modules.videos.models import Video
from modules.videos.video.lib import search_videos
from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import File
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag
from wrolpi.vars import PROJECT_DIR
def test_search_censored_videos(test_session, simple_channel, video_factory):
for i in map(str, range(50)):
video_factory(channel_id=simple_channel.id, source_id=i)
vid = video_factory(source_id=51) # this should never be modified because it has no channel
test_session.add(vid)
test_session.commit()
def set_entries(entries):
simple_channel.info_json = {
'entries': [{'id': j, 'view_count': 0} for j in entries]} if entries else sqlalchemy.null()
test_session.commit()
apply_info_json(simple_channel.id)
test_session.commit()
# All source_ids are in the entries.
set_entries(map(str, range(50)))
videos, total = search_videos(filters=['censored'], order_by='id', limit=20)
assert [i['video']['source_id'] for i in videos] == []
assert total == 0
# First 5 are censored.
set_entries(map(str, range(5, 50)))
videos, total = search_videos(filters=['censored'], order_by='id', limit=20)
assert [i['video']['source_id'] for i in videos] == [str(i) for i in range(5)]
assert all(i['video']['censored'] for i in videos)
assert total == 5
# First 25 are censored.
set_entries(map(str, range(25, 50)))
videos, total = search_videos(filters=['censored'], order_by='id', limit=20)
assert [i['video']['source_id'] for i in videos] == [str(i) for i in range(20)]
assert all(i['video']['censored'] for i in videos)
assert total == 25
@pytest.mark.parametrize('file_name,expected', [
('channel_20000101_12345678910_ some title.mp4', ('channel', '20000101', '12345678910', 'some title')),
('channel name_NA_12345678910_ some title.mp4', ('channel name', None, '12345678910', 'some title')),
@ -75,57 +39,6 @@ def test_parse_video_file_name(file_name, expected):
assert parse_video_file_name(video_path) == expected, f'{file_name} != {expected}'
@pytest.mark.asyncio
async def test_associated_video_files(test_session, test_directory, video_file, image_file, vtt_file1):
"""A Video file has files related to it. They should be attached to the Video record."""
caption_file = video_file.with_suffix('.en.vtt')
vtt_file1.rename(caption_file)
poster_file = video_file.with_suffix('.jpeg')
image_file.rename(poster_file)
await files_lib.refresh_files()
test_session.commit()
# All files were found.
file_paths = {i.path for i in test_session.query(File)}
assert len(file_paths) == 3
video: Video = test_session.query(Video).one()
video_id = video.id
assert video.video_file == video_file
assert video.poster_file == poster_file
assert video.caption_file == caption_file
assert video.video_file.modification_datetime
assert video.video_file.size
assert video.video_file.indexed
await files_lib.refresh_files()
test_session.commit()
# No new files were added.
assert {i.path for i in test_session.query(File)} == file_paths
# Same video exists.
video: Video = test_session.query(Video).one()
assert video.id == video_id, 'Video id changed. Was the video recreated?'
@pytest.mark.asyncio
async def test_unassociated_video(test_session, test_directory, video_file):
"""A Video file without associated files is still a Video."""
# Move the video file into a new subdirectory. The video file should still be found.
sub_dir = test_directory / 'subdir'
sub_dir.mkdir()
new_video_file_path = sub_dir / video_file.name
video_file.rename(new_video_file_path)
video_file = new_video_file_path
await files_lib.refresh_files()
test_session.commit()
video: Video = test_session.query(Video).one()
assert video.video_file == video_file
@pytest.mark.asyncio
async def test_video_factory(test_session, video_factory, channel_factory):
"""
@ -142,35 +55,42 @@ async def test_video_factory(test_session, video_factory, channel_factory):
)
test_session.commit()
assert video.video_path and video.video_file and video.video_path.is_file()
assert video.caption_path and video.caption_file and video.caption_path.is_file()
assert video.poster_path and video.poster_file and video.poster_path.is_file()
assert video.info_json_path and video.info_json_file and video.info_json_path.is_file()
assert video.video_path and video.video_path.is_file()
assert video.caption_paths and all(i.is_file() for i in video.caption_paths)
assert video.poster_path and video.poster_path.is_file()
assert video.info_json_path and video.info_json_path.is_file()
assert video.video_file.a_text # title
assert video.video_file.b_text is None
assert video.video_file.c_text and 'hello' in video.video_file.c_text # description
assert video.video_file.d_text # captions
assert video.file_group.a_text # title
assert video.file_group.b_text is None
assert video.file_group.c_text
assert video.file_group.d_text # captions
assert video.source_id == 'some id'
def test_validate_video(test_session, test_directory, video_factory):
def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file):
"""A video poster will be generated only if the channel permits."""
vid1 = video_factory(with_video_file=True)
video_file = video_file.rename(test_directory / 'Channel Name_20000101_1234567890_The Title.mp4')
vid1 = video_factory(with_video_file=video_file)
# Clear source_id so source_id in the file will be extracted.
vid1.source_id = None
assert not vid1.poster_path
validate_video(vid1, True, test_session)
validate_video(vid1, True)
assert vid1.video_path == video_file
assert vid1.poster_path, 'Poster was not created'
assert vid1.poster_path.is_file(), 'Poster path does not exist'
assert vid1.video_path.stem == vid1.poster_path.stem
assert vid1.poster_path.suffix == '.jpg'
assert vid1.upload_date == datetime.datetime(2000, 1, 1, 0, 1, tzinfo=datetime.timezone.utc)
assert vid1.source_id == '1234567890'
assert vid1.file_group.title == 'The Title'
# A PNG is replaced.
vid2 = video_factory(with_video_file=True, with_poster_ext='.png')
vid2.poster_path.with_suffix('.jpg').touch() # New poster exists, replace it.
vid2.poster_path.with_suffix('.jpg').write_bytes(image_bytes_factory()) # New poster exists, replace it.
assert vid2.poster_path and vid2.poster_path.is_file() and vid2.poster_path.suffix == '.png', \
'Poster was not initialized'
validate_video(vid2, True, test_session)
validate_video(vid2, True)
assert vid2.poster_path.is_file(), 'New poster was not generated'
assert vid2.poster_path.suffix == '.jpg'
@ -197,34 +117,62 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
@pytest.mark.asyncio
async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory):
# A Video without associated files is not orphaned.
vid1 = video_factory(with_video_file=True)
vid1 = video_factory(title='vid1', with_video_file=True)
# The video files will be removed...
vid2 = video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpeg', with_info_json=True)
vid3 = video_factory(with_video_file=True, with_poster_ext='jpg')
vid2 = video_factory(title='vid2', with_video_file=True, with_caption_file=True, with_poster_ext='jpeg',
with_info_json=True)
vid3 = video_factory(title='vid3', with_video_file=True, with_poster_ext='jpg')
# This will be ignored because it is not in the "videos" subdirectory.
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', test_directory / 'video4.en.vtt')
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', test_directory / 'vid4.en.vtt')
test_session.commit()
# Remove vid2 video. Caption, poster, info_json are now orphaned.
vid2.video_path.unlink()
vid2_caption_path, vid2_poster_path, vid2_info_json_path = vid2.caption_path, vid2.poster_path, vid2.info_json_path
vid2_caption_paths, vid2_poster_path, vid2_info_json_path = vid2.caption_paths, vid2.poster_path, vid2.info_json_path
# Remove vid3 video. Poster is now orphaned.
vid3.video_path.unlink()
vid3_poster_path = vid3.poster_path
await refresh_files()
await files_lib.refresh_files()
test_session.commit()
# 6 files when two video files are deleted.
# (vid1, vid2[caption,poster,json], vid3 poster, vid4)
assert test_session.query(File).count() == 6
assert test_session.query(FileGroup).count() == 4
assert test_session.query(Video).count() == 1
videos_directory = test_directory / 'videos'
results = lib.find_orphaned_video_files(videos_directory)
assert sorted(list(results)) == sorted([
vid2_caption_path,
vid2_caption_paths[0],
vid2_info_json_path,
vid2_poster_path,
vid3_poster_path,
])
@pytest.mark.asyncio
async def test_import_channels_config_favorites_tags(test_session, test_directory, test_channels_config, video_factory,
simple_channel):
"""Video favorites have been moved to Tags. Any Favorited video should be tagged with Favorite tag."""
test_session.add(Tag(name='Favorite'))
video1 = video_factory(channel_id=simple_channel.id)
video2 = video_factory(channel_id=simple_channel.id)
test_session.commit()
# Favorite video1 in config.
channels_config = lib.get_channels_config()
channels_config.favorites = {
str(simple_channel.directory.relative_to(test_directory)): {
str(video1.video_path.relative_to(simple_channel.directory)): now().isoformat(),
}
}
# The import function will tag the Video with Favorite.
lib.import_channels_config()
# video1 has Favorite tag. It is also in the config.
assert video1.file_group.tag_files and video1.file_group.tag_files[0].tag.name == 'Favorite'
assert str(video1.file_group.primary_path.relative_to(test_directory)) in test_channels_config.read_text()
# video2 was not tagged.
assert not video2.file_group.tag_files

View file

@ -1,45 +1,73 @@
import pytest
from modules.videos.models import Video
from wrolpi.dates import now
from wrolpi.files.models import File
def test_timezone(test_session):
"""
A Video's dates should have a TimeZone.
"""
video = Video(upload_date=now())
test_session.add(video)
test_session.commit()
assert video.upload_date.tzinfo
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
def test_delete_video_no_channel(test_session, simple_video):
"""
A Video can be deleted even if it does not have a channel.
"""
"""A Video can be deleted even if it does not have a channel."""
assert simple_video.channel
video_id = simple_video.id
simple_video.channel_id = None
test_session.commit()
simple_video: Video = test_session.query(Video).filter_by(id=video_id).one()
paths = list(simple_video.file_group.my_paths())
assert not simple_video.channel
simple_video.delete()
# All files were deleted.
for path in paths:
assert not path.exists()
def test_delete_video(test_session, simple_video, image_file):
"""When a Video record is deleted, all referenced file records should be deleted."""
simple_video.poster_file = File(path=image_file)
test_session.add(simple_video.poster_file)
test_session.commit()
assert simple_video.video_path.is_file(), 'Video file was not created.'
assert simple_video.poster_path.is_file(), 'Video poster was not created.'
assert test_session.query(Video).count() == 1, 'Video was not created.'
assert test_session.query(File).count() == 2, 'Video file and poster file were not created.'
assert test_session.query(FileGroup).count() == 0
assert test_session.query(Video).count() == 0
simple_video.delete()
def test_delete_video(test_session, video_factory):
"""When a Video record is deleted, all referenced file records should be deleted."""
video = video_factory(with_video_file=True, with_poster_ext='png')
test_session.commit()
assert video.video_path.is_file(), 'Video file was not created.'
assert video.poster_path.is_file(), 'Video poster was not created.'
assert test_session.query(Video).count() == 1, 'Video was not created.'
assert test_session.query(FileGroup).count() == 1, 'Video file and poster file were not created.'
video.delete()
test_session.commit()
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
assert test_session.query(File).count() == 0, 'Video files were not deleted.'
assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.'
@pytest.mark.asyncio
async def test_video_channel_refresh(test_session, test_directory, channel_factory, video_factory):
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
# Create a video file in this channel's directory.
channel = channel_factory()
video1_path = channel.directory / 'video1.mp4'
video_factory(channel_id=channel.id, with_video_file=video1_path, with_poster_ext='jpg')
# Create another video outside the channel's directory. It should not have a channel.
video2_path = test_directory / 'video2.mp4'
video_factory(with_video_file=video2_path)
# Delete all FileGroups so the channel will be associated again.
test_session.query(FileGroup).delete()
test_session.commit()
await files_lib.refresh_files()
assert test_session.query(FileGroup).count() == 2
assert test_session.query(Video).count() == 2
# The video is associated with `simple_channel` because its file is in the Channel's directory.
video1: Video = Video.find_by_path(video1_path, test_session)
video2: Video = Video.find_by_path(video2_path, test_session)
assert video1.channel == channel
assert not video2.channel
assert video1.__json__()['video']['channel']
video_channel = video1.__json__()['video']['channel']
assert video_channel['id'] == channel.id
assert video_channel['name'] == channel.name

View file

@ -23,7 +23,7 @@ logger = logger.getChild(__name__)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
def video_get(_: Request, video_id: int):
video, previous_video, next_video = lib.get_video_for_app(video_id)
return json_response({'file': video, 'prev': previous_video, 'next': next_video})
return json_response({'file_group': video, 'prev': previous_video, 'next': next_video})
@video_bp.post('/search')
@ -38,16 +38,16 @@ async def search(_: Request, body: schema.VideoSearchRequest):
if body.order_by not in lib.VIDEO_ORDERS:
raise InvalidOrderBy('Invalid order by')
files, videos_total = lib.search_videos(
file_groups, videos_total = lib.search_videos(
body.search_str,
body.offset,
body.limit,
body.channel_id,
body.order_by,
body.filters,
body.tag_names,
)
ret = {'files': list(files), 'totals': {'files': videos_total}}
ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}
return json_response(ret)

View file

@ -1,15 +1,14 @@
from datetime import datetime
from typing import Tuple, Optional, List
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound
from modules.videos.models import Video
from wrolpi.common import run_after, logger
from wrolpi.db import get_db_session, optional_session
from wrolpi.errors import UnknownVideo
from wrolpi.files.lib import handle_search_results
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
from ..lib import save_channels_config
from ..models import Video
logger.getChild(__name__)
@ -31,47 +30,44 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
video.set_viewed()
previous_video, next_video = video.get_surrounding_videos()
caption = video.video_file.d_text
video = video.video_file.__json__()
video['video']['caption'] = caption
previous_video = previous_video.video_file.__json__() if previous_video and previous_video.video_file else None
next_video = next_video.video_file.__json__() if next_video and next_video.video_file else None
video = video.__json__()
previous_video = previous_video.__json__() if previous_video and previous_video.file_group else None
next_video = next_video.__json__() if next_video and next_video.file_group else None
return video, previous_video, next_video
VIDEO_ORDERS = {
'upload_date': 'v.upload_date ASC, LOWER(v.video_path) ASC',
'-upload_date': 'v.upload_date DESC NULLS LAST, LOWER(v.video_path) DESC',
'rank': '2 DESC, LOWER(v.video_path) DESC',
'-rank': '2 ASC, LOWER(v.video_path) ASC',
'id': 'v.id ASC',
'-id': 'v.id DESC',
'size': 'f.size ASC, LOWER(v.video_path) ASC',
'-size': 'f.size DESC, LOWER(v.video_path) DESC',
'duration': 'duration ASC, LOWER(v.video_path) ASC',
'-duration': 'duration DESC, LOWER(v.video_path) DESC',
'favorite': 'favorite ASC, LOWER(v.video_path) ASC',
'-favorite': 'favorite DESC, LOWER(v.video_path) DESC',
'upload_date': 'v.upload_date ASC, LOWER(fg.primary_path) ASC',
'-upload_date': 'v.upload_date DESC NULLS LAST, LOWER(fg.primary_path) DESC',
'rank': '2 DESC, LOWER(fg.primary_path) DESC',
'-rank': '2 ASC, LOWER(fg.primary_path) ASC',
'id': 'fg.id ASC',
'-id': 'fg.id DESC',
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
'-size': 'fg.size DESC, LOWER(fg.primary_path) DESC',
'duration': 'duration ASC, LOWER(fg.primary_path) ASC',
'-duration': 'duration DESC, LOWER(fg.primary_path) DESC',
'viewed': 'v.viewed ASC',
'-viewed': 'v.viewed DESC',
'view_count': 'v.view_count ASC',
'-view_count': 'v.view_count DESC',
'modification_datetime': 'f.modification_datetime ASC',
'-modification_datetime': 'f.modification_datetime DESC',
'modification_datetime': 'fg.modification_datetime ASC',
'-modification_datetime': 'fg.modification_datetime DESC',
}
NO_NULL_ORDERS = {
'viewed': 'v.viewed IS NOT NULL',
'-viewed': 'v.viewed IS NOT NULL',
'duration': 'v.duration IS NOT NULL',
'-duration': 'v.duration IS NOT NULL',
'size': 'f.size IS NOT NULL',
'-size': 'f.size IS NOT NULL',
'size': 'fg.size IS NOT NULL',
'-size': 'fg.size IS NOT NULL',
'view_count': 'v.view_count IS NOT NULL',
'-view_count': 'v.view_count IS NOT NULL',
'modification_datetime': 'f.modification_datetime IS NOT NULL',
'-modification_datetime': 'f.modification_datetime IS NOT NULL',
'modification_datetime': 'fg.modification_datetime IS NOT NULL',
'-modification_datetime': 'fg.modification_datetime IS NOT NULL',
}
JOIN_ORDERS = ('upload_date', '-upload_date', 'viewed', '-viewed', 'view_count', '-view_count', 'duration', '-duration')
DEFAULT_VIDEO_ORDER = 'rank'
VIDEO_QUERY_LIMIT = 24
@ -81,77 +77,77 @@ def search_videos(
offset: int = None,
limit: int = VIDEO_QUERY_LIMIT,
channel_id: int = None,
order_by: str = None,
filters: List[str] = None,
order: str = None,
tag_names: List[str] = None,
) -> Tuple[List[dict], int]:
filters = filters or []
wheres = ['v.video_path IS NOT NULL']
tag_names = tag_names or []
# Only search videos.
wheres = ["fg.model = 'video'"]
joins = list()
join_video = False
params = dict(search_str=search_str, offset=offset)
if channel_id:
wheres.append('v.channel_id = %(channel_id)s')
joins.append('LEFT JOIN channel c ON c.id = v.channel_id')
join_video = True
params['channel_id'] = channel_id
# Apply filters.
if 'favorite' in filters:
wheres.append('v.favorite IS NOT NULL')
if 'censored' in filters:
wheres.append('v.censored = true')
if search_str:
# A search_str was provided by the user, modify the query to filter by it.
select_columns = 'f.path, ts_rank(f.textsearch, websearch_to_tsquery(%(search_str)s)), ' \
'COUNT(*) OVER() AS total'
wheres.append('f.textsearch @@ websearch_to_tsquery(%(search_str)s)')
select_columns = 'fg.id, ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
params['search_str'] = search_str
join = 'LEFT JOIN file f on f.path = v.video_path'
else:
# No search_str provided. Get path and total only.
select_columns = 'v.video_path AS path, COUNT(*) OVER() AS total'
join = ''
# No search_str provided. Get id and total only.
select_columns = 'fg.id, COUNT(*) OVER() AS total'
if order_by in {'size', '-size', 'modification_datetime', '-modification_datetime'}:
# Size and modification_datetime are from the video file.
join = 'LEFT JOIN file f on f.path = v.video_path'
if tag_names:
where_, params_, join_ = tag_names_to_clauses(tag_names)
wheres.append(where_)
params.update(params_)
joins.append(join_)
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
# whitelist.
order = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
if order_by:
order_by = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
if order:
try:
order = VIDEO_ORDERS[order_by]
order_by = VIDEO_ORDERS[order]
except KeyError:
raise
if order_by in NO_NULL_ORDERS:
wheres.append(NO_NULL_ORDERS[order_by])
if order in NO_NULL_ORDERS:
wheres.append(NO_NULL_ORDERS[order])
if order in JOIN_ORDERS:
join_video = True
if join_video:
joins.insert(0, 'LEFT JOIN video v on v.file_group_id = fg.id')
wheres = '\n AND '.join(wheres)
where = f'WHERE\n{wheres}' if wheres else ''
join = '\n'.join(joins)
stmt = f'''
SELECT
{select_columns}
FROM video v
FROM file_group fg
{join}
{where}
ORDER BY {order}
ORDER BY {order_by}
OFFSET %(offset)s LIMIT {int(limit)}
'''.strip()
logger.debug(stmt, params)
results, total = handle_search_results(stmt, params)
results, total = handle_file_group_search_results(stmt, params)
return results, total
@run_after(save_channels_config)
def set_video_favorite(video_id: int, favorite: bool) -> Optional[datetime]:
"""
Set the Video.favorite to the current datetime if `favorite` is True, otherwise None.
"""
with get_db_session(commit=True) as session:
video = session.query(Video).filter_by(id=video_id).one()
favorite = video.set_favorite(favorite)
return favorite
def tag_video(video_id: int, tag_name: str):
"""Tag a video"""
with get_db_session(commit=True):
video = Video.find_by_id(video_id)
video.add_tag(tag_name)
@optional_session

View file

@ -1,3 +1,4 @@
import json
from datetime import timedelta
from http import HTTPStatus
from json import dumps
@ -60,19 +61,19 @@ def test_get_video_prev_next(test_session, channel_factory, video_factory):
if prev_title is None:
assert prev_video is None
else:
assert prev_video and prev_video.title == prev_title
assert prev_video and prev_video.file_group.title == prev_title
if next_title is None:
assert next_video is None
else:
assert next_video and next_video.title == next_title
assert next_video and next_video.file_group.title == next_title
except AssertionError as e:
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
def test_get_video_for_app(test_session, simple_channel, simple_video):
vid, prev, next_ = get_video_for_app(simple_video.id)
assert vid['video']['id'] == simple_video.id
assert vid['id'] == simple_video.id
@pytest.mark.asyncio
@ -85,7 +86,7 @@ def test_video_delete(test_client, test_session, test_directory, channel_factory
assert test_session.query(Video).count() == 2
vid1_video_path, vid1_caption_path = vid1.video_path, vid1.caption_path
vid1_video_path, vid1_caption_path = vid1.video_path, vid1.caption_paths[0]
vid2_video_path, vid2_info_json_path = vid2.video_path, vid2.info_json_path
# No videos have been deleted yet.
@ -120,10 +121,10 @@ def test_video_delete(test_client, test_session, test_directory, channel_factory
assert response.status_code == HTTPStatus.BAD_REQUEST
def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixture, test_download_manager):
def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixture, test_download_manager, tag_factory):
"""Many methods are blocked when WROL Mode is enabled."""
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
favorite = dumps(dict(video_id=simple_video.id, favorite=True))
tag = tag_factory()
wrol_mode_fixture(True)
@ -152,8 +153,9 @@ def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixtu
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
# THE REST OF THESE METHODS ARE ALLOWED
_, resp = api_app.test_client.post('/api/videos/favorite', content=favorite)
assert resp.status_code == HTTPStatus.OK
content = dict(video_id=simple_video.id, tag_name=tag.name)
_, resp = api_app.test_client.post('/api/videos/tag', content=json.dumps(content))
assert resp.status_code == HTTPStatus.NO_CONTENT
assert test_download_manager.stopped.is_set()

View file

@ -11,9 +11,7 @@ formatters = {
def video_url_resolver(domain: str, entry: dict) -> str:
"""
Guess the full URL of a Youtube-DL entry.
"""
"""Guess the full URL of a Youtube-DL entry."""
if 'webpage_url' in entry:
return entry['webpage_url']

View file

@ -2,7 +2,7 @@ Pillow==9.3.0
Pint==0.18
PyPDF2[crypto]==2.11.2
PyYAML==6.0
SQLAlchemy==1.3.22
SQLAlchemy[asyncio]==1.3.22
aiohttp==3.8.3
alembic==1.7.5
beautifulsoup4==4.10.0

View file

@ -36,14 +36,40 @@ from wrolpi.vars import PYTEST, MODULES_DIR, \
logger = logging.getLogger()
ch = logging.StreamHandler()
formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s')
formatter = logging.Formatter('[%(asctime)s] [%(name)s:%(lineno)d] [%(levelname)s] %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
logger_ = logger.getChild(__name__)
def set_log_level(level, warn_level: bool = True):
"""Set the level of the root logger so all children that have been created (or will be created) share the same
level."""
logger.setLevel(level)
if level == logging.NOTSET:
# Custom maximum logging level which will include SQLAlchemy debugging.
sa_logger = logging.getLogger('sqlalchemy.engine')
sa_logger.setLevel(logging.DEBUG)
# Always warn about the log level, so we know what should have been logged.
effective_level = logger.getEffectiveLevel()
level_name = logging.getLevelName(effective_level)
if warn_level:
logger.warning(f'Logging level: {level_name}')
@contextlib.contextmanager
def log_level_context(level):
starting_level = logger.getEffectiveLevel()
set_log_level(level, warn_level=False)
yield
set_log_level(starting_level, warn_level=False)
__all__ = [
'logger',
'set_log_level',
'Base',
'ModelHelper',
'get_model_by_table_name',
@ -95,7 +121,6 @@ __all__ = [
'truncate_object_bytes',
'background_task',
'get_warn_once',
'ordered_unique_list',
'truncate_generator_bytes',
'cancel_refresh_tasks',
'cancelable_wrapper',
@ -123,8 +148,8 @@ class ModelHelper:
def find_by_paths(paths, session) -> List:
raise NotImplementedError('This model has not defined this method.')
@property
def primary_path(self):
@staticmethod
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
raise NotImplementedError('This model has not defined this method.')
@ -622,7 +647,7 @@ def api_param_limiter(maximum: int, default: int = 20) -> callable:
>>> limiter(100)
100
>>> limiter(150)
150
100
"""
def limiter_(i: int) -> int:
@ -732,12 +757,14 @@ def walk(path: Path) -> Generator[Path, None, None]:
yield from walk(path)
def get_files_and_directories(path: Path):
def get_files_and_directories(directory: Path):
"""Walk a directory, return a tuple of all files and all directories within. (Not recursive)."""
if not path.is_dir():
if not directory.exists():
raise ValueError(f'Directory does not exist: {directory}')
if not directory.is_dir():
raise ValueError('Can only walk a directory.')
directories, files = partition(lambda i: i.is_dir(), path.iterdir())
directories, files = partition(lambda i: i.is_dir(), directory.iterdir())
return files, directories
@ -798,27 +825,11 @@ def register_modeler(modeler: callable):
return modeler
def apply_modelers(files, session: Session):
"""
Group Files by stem, send those groups to the modelers. Each modeler will model what Files are valid.
"""
from wrolpi.files.lib import split_path_stem_and_suffix
if not files:
return
# Group all files by their common name (without the suffix).
groups = {}
for file in files:
file.do_stats()
stem, _ = split_path_stem_and_suffix(file.path)
try:
groups[stem].append(file)
except KeyError:
groups[stem] = [file, ]
async def apply_modelers():
for modeler in modelers:
modeler(groups, session)
await modeler()
# Sleep to catch cancel.
await asyncio.sleep(0)
after_refresh = []
@ -829,7 +840,7 @@ def register_after_refresh(func: callable):
return func
def apply_after_refresh():
async def apply_after_refresh():
for func in after_refresh:
logger_.info(f'Applying after-refresh {func.__name__}')
func()
@ -871,7 +882,7 @@ def chunks_by_stem(it: List[Union[pathlib.Path, str]], size: int) -> Generator[L
raise ValueError('size must be a positive integer')
if not it or len(it) < size:
yield it
yield sorted(it)
return
if not isinstance(it, list):
@ -1042,11 +1053,6 @@ def get_warn_once(message: str, logger__: logging.Logger, level=logging.ERROR):
return warn_once
def ordered_unique_list(lst: Iterable) -> List:
"""Return a new list that contains only the first occurrence of each item."""
return list(dict.fromkeys(lst))
def get_global_statistics():
from wrolpi.db import get_db_curs
with get_db_curs() as curs:

View file

@ -2,10 +2,14 @@
Fixtures for Pytest tests.
"""
import asyncio
import copy
import json
import logging
import multiprocessing
import pathlib
import shutil
import tempfile
from itertools import zip_longest
from typing import List, Callable, Union, Dict
from typing import Tuple, Set
from unittest import mock
@ -13,18 +17,21 @@ from unittest.mock import MagicMock
from uuid import uuid1, uuid4
import pytest
import sqlalchemy
from PIL import Image
from sanic_testing.testing import SanicTestClient
from sqlalchemy.engine import Engine, create_engine
from sqlalchemy.orm import Session, sessionmaker
from wrolpi import flags
from wrolpi.common import iterify
from wrolpi.common import iterify, log_level_context
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download
from wrolpi.files import lib as files_lib
from wrolpi.root_api import BLUEPRINTS, api_app
from wrolpi.tags import Tag
from wrolpi.vars import PROJECT_DIR
@ -75,6 +82,13 @@ def test_session() -> Session:
conn.close()
@pytest.fixture(autouse=True)
def test_debug_logger(level: int = logging.DEBUG):
"""Tests use debug logging by default."""
with log_level_context(level):
yield
@pytest.fixture(autouse=True)
def test_directory() -> pathlib.Path:
"""
@ -180,6 +194,19 @@ def video_file(test_directory) -> pathlib.Path:
yield destination
@pytest.fixture
def video_file_factory(test_directory):
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
def _(path: pathlib.Path = None) -> pathlib.Path:
destination = path or test_directory / f'{uuid4()}.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
return destination
return _
@pytest.fixture
def corrupted_video_file(test_directory) -> pathlib.Path:
"""Return a copy of the corrupted video file in the `test_directory`."""
@ -334,3 +361,171 @@ def flags_lock():
finally:
flags.TESTING_LOCK.clear()
FLAGS_LOCK.release()
@pytest.fixture()
def tag_factory(test_session):
names = ['one', 'two', 'three', 'four', 'five', 'six']
count = 1
def factory() -> Tag:
tag = Tag(name=names.pop(0), color=f'#{str(count) * 6}')
test_session.add(tag)
test_session.commit()
return tag
return factory
@pytest.fixture
def assert_files(test_session, test_directory):
def _(files: List[Dict]):
from wrolpi.files.models import FileGroup
for expected in files:
from wrolpi.files.lib import split_path_stem_and_suffix
full_stem, _ = split_path_stem_and_suffix(test_directory / expected.pop('path'), full=True)
try:
file: FileGroup = test_session.query(FileGroup).filter_by(full_stem=full_stem).one()
except sqlalchemy.orm.exc.NoResultFound as e:
raise ValueError(f'No FileGroup found with {full_stem=}') from e
for key, value in expected.items():
attr = getattr(file, key)
if attr != value:
raise AssertionError(f'{full_stem} {attr} != {value}')
return _
@pytest.fixture
def assert_file_groups(test_session, test_directory):
from wrolpi.test.common import assert_dict_contains
def _(file_groups: List[Dict], assert_count: bool = True):
from wrolpi.files.models import FileGroup
for expected in file_groups:
if 'full_stem' in expected:
full_stem = str(test_directory / expected.pop('full_stem'))
try:
file_group: FileGroup = test_session.query(FileGroup).filter_by(full_stem=full_stem).one()
except sqlalchemy.exc.NoResultFound as e:
raise ValueError(f'No FileGroup found with {full_stem=}') from e
elif 'primary_path' in expected:
primary_path = str(test_directory / expected.pop('primary_path'))
try:
file_group: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
except sqlalchemy.exc.NoResultFound as e:
raise ValueError(f'No FileGroup found with {primary_path=}') from e
# Compare file dictionaries.
files = expected.pop('files', None)
if files:
for exp, actual in zip_longest(files, file_group.files):
assert_dict_contains(actual, exp)
for key, value in expected.items():
attr = getattr(file_group, key)
if isinstance(value, dict):
assert_dict_contains(attr, value)
elif attr != value:
raise AssertionError(f'{file_group}.{key} {attr=} != {value}')
if assert_count is True:
if (count := test_session.query(FileGroup).count()) != len(file_groups):
raise AssertionError(f'FileGroup count does not match {count} != {len(file_groups)}')
return _
@pytest.fixture
def assert_files_search(test_client):
from wrolpi.test.common import assert_dict_contains
def _(search_str: str, expected: List[dict]):
content = json.dumps({'search_str': search_str})
request, response = test_client.post('/api/files/search', content=content)
for file_group, exp in zip_longest(response.json['file_groups'], expected):
assert_dict_contains(file_group, exp)
assert len(response.json['file_groups']) == len(expected)
return _
SINGLEFILE_CONTENTS = '''<html><!--
Page saved with SingleFile
url: {url}
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">
<meta name="msapplication-TileColor" content="#2d89ef">
<meta name="theme-color" content="#000000">
{title}
<body>
some body contents
</body>
</html>
'''
@pytest.fixture
def singlefile_contents_factory():
def _(title: str = '', url: str = 'https://example.com') -> str:
contents = copy.deepcopy(SINGLEFILE_CONTENTS)
contents = contents.format(
title=f'<title>{title}</title>',
url=url,
)
return contents
return _
@pytest.fixture
def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.Path:
archive = test_directory / 'archive'
archive.mkdir(exist_ok=True)
singlefile = archive / 'singlefile.html'
singlefile.write_text(singlefile_contents_factory())
return singlefile
@pytest.fixture
def image_bytes_factory():
def _() -> bytes:
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
Image.new('RGB', (1, 1), color='grey').save(fh)
fh.seek(0)
image = fh.read()
return image
return _
@pytest.fixture
def insert_file_group(test_session, test_directory):
"""Inserts a FileGroup based on the provided paths. Does not verify if the files exist."""
def _(paths: List[pathlib.Path]):
files = [dict(path=str(i), mimetype='fake') for i in paths]
params = dict(
full_stem=str(test_directory / files_lib.split_path_stem_and_suffix(paths[0])[0]),
primary_path=str(paths[0]),
files=json.dumps(files),
)
test_session.execute('INSERT INTO file_group '
'(indexed, full_stem, primary_path, files) VALUES '
'(true, :full_stem, :primary_path, :files)', params)
return _
@pytest.fixture
def test_tags_config(test_directory):
from wrolpi.tags import test_tags_config
with test_tags_config():
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/tags.yaml'
yield config_path

View file

@ -96,7 +96,7 @@ class TZDateTime(types.TypeDecorator):
def process_bind_param(self, value, dialect):
if isinstance(value, datetime):
if not value.tzinfo:
raise TypeError("tzinfo is required")
raise TypeError(f"tzinfo is required: {value}")
# Store all timestamps as UTC timezone.
value = value.astimezone(timezone.utc)
elif isinstance(value, str) and '-' in value:

View file

@ -1,9 +1,13 @@
import pathlib
import types
from contextlib import contextmanager
from functools import wraps
from typing import ContextManager, Tuple, List, Union, Type
from typing import ContextManager, Tuple, List, Union, Type, Generator
import psycopg2
import psycopg2.extensions
import sqlalchemy.exc
from psycopg2._psycopg import cursor
from sqlalchemy import create_engine, inspect
from sqlalchemy.orm import sessionmaker, Session
from sqlalchemy.pool import NullPool
@ -166,3 +170,12 @@ def get_ranked_models(ranked_primary_keys: List, model: Type[Base], session: Ses
results = list(session.query(model).filter(pkey.in_(ranked_primary_keys)).all())
results = sorted(results, key=lambda i: ranked_primary_keys.index(getattr(i, pkey_name)))
return results
def mogrify(curs: cursor, values: Union[List, Generator]) -> str:
values = list(values) if isinstance(values, types.GeneratorType) else values.copy()
count = len(values[0])
# If `count = 3`, then s = '(%s,%s,%s)'
s = '(' + ','.join("%s" for _ in range(count)) + ')'
result = ',\n'.join([curs.mogrify(s, i).decode() for i in values])
return result

View file

@ -152,6 +152,14 @@ class UnknownArchive(APIError):
pass
class UnknownTag(APIError):
pass
class UsedTag(APIError):
pass
error_codes = iter(range(1, 1000))
API_ERRORS = {
@ -334,5 +342,15 @@ API_ERRORS = {
'code': next(error_codes),
'message': 'Unable to find the archive',
'status': HTTPStatus.NOT_FOUND,
}
},
UnknownTag: {
'code': next(error_codes),
'message': 'Unable to find the tag',
'status': HTTPStatus.NOT_FOUND,
},
UsedTag: {
'code': next(error_codes),
'message': 'Tag is being used',
'status': HTTPStatus.BAD_REQUEST,
},
}

View file

@ -29,13 +29,17 @@ class Events:
def send_global_refresh_modeling_completed(message: str = None):
send_event('global_refresh_modeling_completed', message, subject='refresh')
@staticmethod
def send_global_refresh_discovery_completed(message: str = None):
send_event('global_refresh_discovery_completed', message, subject='refresh')
@staticmethod
def send_global_refresh_indexing_completed(message: str = None):
send_event('global_refresh_indexing_completed', message, subject='refresh')
@staticmethod
def send_global_refresh_delete_completed(message: str = None):
send_event('global_refresh_delete_completed', message, subject='refresh')
def send_global_after_refresh_completed(message: str = None):
send_event('global_after_refresh_completed', message, subject='refresh')
@staticmethod
def send_ready(message: str = None):

View file

@ -1,18 +1,14 @@
import pathlib
from http import HTTPStatus
from typing import List
from urllib.request import Request
from sanic import response
from sanic import response, Request
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import get_media_directory, wrol_mode_check, background_task, get_relative_to_media_directory
from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory
from wrolpi.errors import InvalidFile
from wrolpi.root_api import get_blueprint, json_response
from . import lib, schema
from ..schema import JSONErrorResponse
from ..vars import PYTEST
bp = get_blueprint('Files', '/api/files')
@ -44,38 +40,20 @@ async def delete_file(_: Request, body: schema.DeleteRequest):
@bp.post('/refresh')
@openapi.description('Find and index all files in the media directory.')
@wrol_mode_check
async def refresh(_: Request):
await lib.refresh_files()
return response.empty()
@bp.post('/refresh/directory')
@openapi.description('Find and index all files in the provided directory.')
@validate(schema.DirectoryRefreshRequest)
@wrol_mode_check
async def refresh_directory(_: Request, body: schema.DirectoryRefreshRequest):
directory = get_media_directory() / body.directory
if PYTEST:
await lib.refresh_directory_files_recursively(directory)
else:
background_task(lib.refresh_directory_files_recursively(directory))
return response.empty()
@bp.post('/refresh/list')
@openapi.definition(
summary='Refresh and index all files in the provided list',
body=schema.FilesRefreshListRequest,
summary='Refresh and index all paths (files/directories) in the provided list. Refresh all files if not provided.',
body=schema.FilesRefreshRequest,
)
@validate(schema.FilesRefreshListRequest)
@wrol_mode_check
async def refresh_files_list(_: Request, body: schema.FilesRefreshListRequest):
if PYTEST:
await lib.refresh_files_list(body.files, body.include_files_near)
else:
background_task(lib.refresh_files_list(body.files, body.include_files_near))
async def refresh(request: Request):
paths = None
if request.body:
media_directory = get_media_directory()
if not isinstance(request.json['paths'], list):
raise ValueError('Can only refresh a list')
paths = [media_directory / i for i in request.json['paths']]
await lib.refresh_files(paths)
return response.empty()
@ -86,8 +64,9 @@ async def refresh_files_list(_: Request, body: schema.FilesRefreshListRequest):
)
@validate(schema.FilesSearchRequest)
async def post_search_files(_: Request, body: schema.FilesSearchRequest):
files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model)
return json_response(dict(files=files, totals=dict(files=total)))
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
body.tag_names)
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))
@bp.post('/directories')
@ -106,3 +85,17 @@ def post_directories(_, body: schema.DirectoriesRequest):
body = {'directories': dirs, 'exists': path.exists(), 'is_dir': path.is_dir(), 'is_file': path.is_file()}
return response.json(body)
@bp.post('/tag')
@validate(schema.TagFileGroupPost)
def post_tag_file_group(_, body: schema.TagFileGroupPost):
lib.add_file_group_tag(body.file_group_id, body.tag_name)
return response.empty(HTTPStatus.CREATED)
@bp.post('/untag')
@validate(schema.TagFileGroupPost)
def post_untag_file_group(_, body: schema.TagFileGroupPost):
lib.remove_file_group_tag(body.file_group_id, body.tag_name)
return response.empty(HTTPStatus.NO_CONTENT)

View file

@ -1,17 +1,18 @@
import asyncio
import dataclasses
import logging
import pathlib
from typing import Dict, List, Generator, Optional
from typing import List, Optional, Tuple
import bs4
from sqlalchemy import Column, Integer, ForeignKey, String
from sqlalchemy.orm import Session, relationship
from sqlalchemy import Column, Integer, String, BigInteger, ForeignKey, or_, and_
from sqlalchemy.orm import relationship
from wrolpi.common import register_modeler, ModelHelper, Base, register_after_refresh
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.common import ModelHelper, Base, register_modeler
from wrolpi.db import get_db_session
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
from wrolpi.files.models import File
from wrolpi.media_path import MediaPathType
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
try:
import ebooklib
@ -48,10 +49,21 @@ class EBookData:
creator: str = None
text: str = None
title: str = None
ebook_path: pathlib.Path = None
def __bool__(self):
return bool(self.cover) or bool(self.creator) or bool(self.text) or bool(self.title)
def __json__(self):
d = {}
if self.creator:
d['creator'] = self.creator
if self.title:
d['title'] = self.title
if self.ebook_path:
d['ebook_path'] = self.ebook_path
return d
def extract_text(html: str) -> str:
soup = bs4.BeautifulSoup(html, features='html.parser')
@ -59,8 +71,8 @@ def extract_text(html: str) -> str:
return text
def mimetype_is_ebook(mimetype: str) -> bool:
return any(mimetype.startswith(i) for i in EBOOK_MIMETYPES)
def mimetype_is_ebook(mimetype: str) -> List[str]:
return [i for i in EBOOK_MIMETYPES if mimetype.startswith(i)]
def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]:
@ -70,7 +82,7 @@ def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]
if not mimetype.startswith(EPUB_MIMETYPE):
return None
data = EBookData()
data = EBookData(ebook_path=path)
book = epub.read_epub(path, options=dict(ignore_ncx=True))
for key, value in book.metadata.items():
@ -139,53 +151,25 @@ class EBook(ModelHelper, Base):
id = Column(Integer, primary_key=True)
size: int = Column(Integer)
title: str = Column(String)
creator: str = Column(String)
ebook_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
ebook_file: File = relationship('File', primaryjoin='EBook.ebook_path==File.path')
cover_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
cover_file: File = relationship('File', primaryjoin='EBook.cover_path==File.path')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
file_group: FileGroup = relationship('FileGroup')
def __repr__(self):
cover = 'yes' if self.cover_file else 'no'
return f'<EBook id={self.id} path={repr(self.ebook_path)} cover={cover}>'
path = str(self.file_group.primary_path)
return f'<EBook id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
def my_paths(self) -> Generator[pathlib.Path, None, None]:
if self.ebook_path:
yield self.ebook_path
if self.cover_path:
yield self.cover_path
def my_files(self) -> Generator[File, None, None]:
if self.ebook_file:
yield self.ebook_file
if self.cover_file:
yield self.cover_file
@staticmethod
def find_by_path(path, session) -> Base:
return session.query(EBook).filter(EBook.ebook_path == path).one_or_none()
@staticmethod
def find_by_paths(paths: List[pathlib.Path], session) -> List:
ebooks = list(session.query(EBook).filter(EBook.ebook_path.in_(paths)))
return ebooks
def __json__(self):
d = dict(
cover_path=self.cover_path,
creator=self.creator,
ebook_path=self.ebook_path,
id=self.id,
size=self.size,
title=self.title,
)
def __json__(self) -> dict:
# Convert this eBook's paths to pathlib.Paths.
d = self.file_group.__json__()
d['data']['ebook_path'] = pathlib.Path(d['data']['ebook_path'])
d['data']['cover_path'] = pathlib.Path(d['data']['cover_path']) if 'cover_path' in d['data'] else None
return d
def generate_cover(self) -> Optional[pathlib.Path]:
"""Discover this ebook's cover, if found, store it next to the ebook file."""
path = self.ebook_file.path if self.ebook_file else self.ebook_path
path = self.file_group.primary_path
if not path:
raise ValueError('Cannot generate cover for ebook without ebook file!')
@ -201,113 +185,110 @@ class EBook(ModelHelper, Base):
logger.warning(f'Unable to generate cover for {self.ebook_path}')
@property
def cover_file(self) -> Optional[dict]:
posters = self.file_group.my_poster_files()
return posters[0]
def _model_ebook(ebook: EBook, files: List[File]) -> EBook:
@property
def cover_path(self) -> Optional[pathlib.Path]:
return self.cover_file['path']
def _model_ebook(ebook: EBook) -> EBook:
"""Creates an EBook model based off a File. Searches for it's cover in the provided `files`."""
# Multiple formats may share this cover.
cover_file = next((i for i in files if i.mimetype.split('/')[0] == 'image'), None)
epub_file = next(iter(ebook.file_group.my_files('application/epub')), None)
mobi_file = next(iter(ebook.file_group.my_files('application/x-mobipocket-ebook')), None)
# epub is preferred because it can be indexed.
if epub_file:
ebook_file = epub_file['path']
ebook_mimetype = epub_file['mimetype']
elif mobi_file:
ebook_file = mobi_file['path']
ebook_mimetype = mobi_file['mimetype']
else:
raise ValueError(f'No epub or mobi found. {ebook}')
ebook_file = ebook.ebook_file
size = ebook_file.path.stat().st_size
ebook.ebook_file.do_index()
size = ebook_file.stat().st_size
# Only index if it hasn't been done, or if the file has changed.
changed = size != ebook.size or cover_file != ebook.cover_file or not ebook.title
if not ebook.ebook_file.indexed or changed:
changed = size != ebook.size or not ebook.title
if epub_file and changed:
# Only read the contents of the file if it has changed.
try:
# Only read the contents of the file if it has changed.
data = extract_ebook_data(ebook_file.path, ebook_file.mimetype)
data = extract_ebook_data(ebook_file, ebook_mimetype)
if data:
# Title is a_text.
ebook.ebook_file.a_text = ebook.title = data.title
ebook.file_group.title = ebook.file_group.a_text = ebook.title = data.title
# Creator is b_text.
ebook.ebook_file.b_text = ebook.creator = data.creator
ebook.file_group.b_text = ebook.creator = data.creator
# All text is d_text.
ebook.ebook_file.d_text = data.text
ebook.file_group.d_text = data.text
d: dict = data.__json__()
d['ebook_path'] = str(d['ebook_path'])
ebook.file_group.data = d
except Exception as e:
logger.error(f'Failed to extract book data', exc_info=e)
logger.error(f'Failed to extract epub book data', exc_info=e)
if not ebook.title:
if ebook.file_group.my_poster_files():
ebook.file_group.data['cover_path'] = ebook.file_group.my_poster_files()[0]['path']
else:
# Extract cover because there is no cover file.
cover_path = ebook.generate_cover()
ebook.file_group.append_files(cover_path)
ebook.file_group.data['cover_path'] = str(cover_path)
if not ebook.file_group.title:
# Book was not indexed above (probably a MOBI), use the file data.
stem, _ = split_path_stem_and_suffix(ebook.ebook_file.path)
ebook.title = ebook.ebook_file.a_text = stem
stem, _ = split_path_stem_and_suffix(ebook.file_group.primary_path)
ebook.file_group.title = stem
if cover_file:
cover_file.associated = True
ebook.cover_file = cover_file
ebook.size = size
ebook_file.model = EBook.__tablename__
return ebook
def find_ebook_files_in_group(group: List[File]):
def find_ebook_files_in_group(group: List):
"""Returns a list of any File records if they are an ebook."""
return list(filter(lambda i: mimetype_is_ebook(i.mimetype), group))
@register_modeler
def ebook_modeler(groups: Dict[str, List[File]], session: Session):
async def ebook_modeler():
"""Searches for ebook files and models them into the ebook table."""
# Get all ebook files.
ebook_files = {stem: files for stem, group in groups.items() if (files := find_ebook_files_in_group(group))}
if not ebook_files:
# No ebooks in these groups.
return
# Get all ebooks records (if any) in one query.
ebook_paths = [j.path for i in ebook_files.values() for j in i]
ebook_records = {i.ebook_path: i for i in session.query(EBook).filter(EBook.ebook_path.in_(ebook_paths))}
for stem, ebook_files in ebook_files.items():
group = groups[stem]
session.flush(group)
# Process each ebook file in the group. There may be more than one format in each group, and they may all
# share a cover.
for ebook_file in ebook_files:
ebook: EBook = ebook_records.get(ebook_file.path)
if not ebook:
ebook = EBook(ebook_file=ebook_file)
session.add(ebook)
_model_ebook(ebook, group)
# Remove this group, it will not be processed by other modelers.
del groups[stem]
@register_after_refresh
def discover_ebook_covers():
"""Discovers/extracts ebook covers for any Ebook that is missing one."""
with get_db_curs() as curs:
stmt = f'''
SELECT e.id
FROM ebook e
LEFT JOIN file f on e.ebook_path = f.path
WHERE
e.cover_path IS NULL
AND f.mimetype LIKE '{EPUB_MIMETYPE}%' '''
curs.execute(stmt)
missing_covers = [i[0] for i in curs.fetchall()]
if not missing_covers:
logger.info('No ebook covers to generate')
return
logger.warning(f'Discovering/generating {len(missing_covers)} ebook covers')
for id_ in missing_covers:
while True:
with get_db_session(commit=True) as session:
ebook: EBook = session.query(EBook).filter(EBook.id == id_).one()
file_groups = session.query(FileGroup, EBook).filter(
or_(
FileGroup.mimetype == 'application/epub+zip',
FileGroup.mimetype == 'application/x-mobipocket-ebook',
),
FileGroup.indexed == False,
).outerjoin(EBook, and_(EBook.file_group_id == FileGroup.id)) \
.limit(10)
file_groups: List[Tuple[FileGroup, EBook]] = list(file_groups)
cover_path = ebook.generate_cover()
if cover_path:
# Cover was generated or discovered. Associate it with the ebook.
cover_file = session.query(File).filter(File.path == cover_path).one_or_none()
if not cover_file:
cover_file = File(path=cover_path)
session.add(cover_file)
cover_file.do_index()
cover_file.associated = True
processed = 0
for file_group, ebook in file_groups:
processed += 1
try:
ebook = ebook or EBook(file_group=file_group)
if PYTEST:
# Handle caching issue during testing.
session.expire(file_group)
_model_ebook(ebook)
session.add(ebook)
file_group.model = EBook.__tablename__
file_group.indexed = True
except Exception as e:
logger.error(f'Failed to index ebook {file_group}')
if PYTEST:
raise
ebook.cover_file = cover_file
if processed < 10:
# Did not reach limit, no more books.
break
# Sleep to catch cancel.
await asyncio.sleep(0)

View file

@ -1,18 +1,17 @@
import dataclasses
import pathlib
from abc import ABC
from collections import defaultdict
from typing import List, Type, Tuple, Generator
from typing import List, Type, Tuple
from zipfile import ZipFile
from wrolpi.vars import PYTEST, FILE_MAX_PDF_SIZE, FILE_MAX_TEXT_SIZE
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
try:
from PyPDF2 import PdfReader
except ImportError:
PdfReader = None
from wrolpi.common import logger, truncate_generator_bytes, split_lines_by_length, truncate_object_bytes
from wrolpi.common import logger, split_lines_by_length, truncate_object_bytes
logger = logger.getChild(__name__)
@ -22,16 +21,15 @@ __all__ = ['Indexer', 'DefaultIndexer', 'ZipIndexer', 'TextIndexer', 'find_index
class Indexer(object):
@staticmethod
def get_title(file):
def get_title(path: pathlib.Path):
from wrolpi.files.lib import split_file_name_words
path = file.path.path if hasattr(file.path, 'path') else file.path
words = split_file_name_words(path.name)
return words
@classmethod
def create_index(cls, file) -> Tuple:
def create_index(cls, path: pathlib.Path) -> Tuple:
# All files can have their title as the highest priority search.
a = cls.get_title(file)
a = cls.get_title(path)
return a, None, None, None
@ -43,17 +41,17 @@ class DefaultIndexer(Indexer, ABC):
indexer_map = defaultdict(lambda: DefaultIndexer)
def find_indexer(file) -> Type[Indexer]:
def find_indexer(mimetype: str) -> Type[Indexer]:
"""Find the Indexer for a given File model."""
if not file.mimetype:
if not mimetype:
return DefaultIndexer
# Get the indexer that matches the mimetype of this file.
indexer = next((v for k, v in indexer_map.items() if file.mimetype.startswith(k)), None)
indexer = next((v for k, v in indexer_map.items() if mimetype.startswith(k)), None)
if not indexer:
# Use the broad indexer.
indexer = indexer_map[file.mimetype.split('/')[0]]
indexer = indexer_map[mimetype.split('/')[0]]
return indexer
@ -77,19 +75,18 @@ class ZipIndexer(Indexer, ABC):
"""Handles archive files like zip."""
@classmethod
def create_index(cls, file):
a = cls.get_title(file)
file_names = cls.get_file_names(file)
def create_index(cls, path: pathlib.Path):
a = cls.get_title(path)
file_names = cls.get_file_names(path)
return a, None, None, file_names
@classmethod
def get_file_names(cls, file) -> List[str]:
def get_file_names(cls, path: pathlib.Path) -> List[str]:
"""Return all the names of the files in the zip file."""
path = file.path.path if hasattr(file.path, 'path') else file.path
from wrolpi.files.lib import split_file_name_words
try:
with ZipFile(path, 'r') as zip_:
file_names = [' '.join(split_file_name_words(pathlib.Path(name).name)) for name in zip_.namelist()]
file_names = ' '.join([split_file_name_words(pathlib.Path(name).name) for name in zip_.namelist()])
file_names = truncate_object_bytes(file_names, FILE_MAX_TEXT_SIZE)
return file_names
except Exception as e:
@ -105,9 +102,8 @@ class TextIndexer(Indexer, ABC):
Detects VTT (caption) files and forwards them on."""
@classmethod
def create_index(cls, file):
path = file.path.path if hasattr(file.path, 'path') else file.path
a = cls.get_title(file)
def create_index(cls, path: pathlib.Path):
a = cls.get_title(path)
words = cls.get_words(path)
return a, None, None, words
@ -119,78 +115,3 @@ class TextIndexer(Indexer, ABC):
contents = truncate_object_bytes(contents, FILE_MAX_TEXT_SIZE)
words = split_lines_by_length(contents)
return words
@dataclasses.dataclass
class PDFMetadata:
title: str = None
author: str = None
@register_indexer('application/pdf')
class PDFIndexer(Indexer, ABC):
"""Uses PyPDF2 to extract text from a PDF."""
@classmethod
def create_index(cls, file):
path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
file_title = super().get_title(file)
reader = None
if PdfReader is not None:
reader = PdfReader(path)
data = cls.get_pdf_metadata(reader, file)
if path.stat().st_size > FILE_MAX_PDF_SIZE:
logger.warning(f'PDF too large to fully index: {path}')
return data.title, data.author, file_title, None
words = ''
try:
# PDFs are complex, don't fail to create title index if text extraction fails.
words = '\n'.join(truncate_generator_bytes(cls.get_words(reader, path), FILE_MAX_TEXT_SIZE))
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
words = split_lines_by_length(words)
except Exception as e:
logger.error(f'Failed to index {path}', exc_info=e)
if PYTEST:
raise
return data.title, data.author, file_title, words
@classmethod
def get_pdf_metadata(cls, reader: PdfReader, file) -> PDFMetadata:
"""Extract title/author from the PDF metadata."""
path = file.path.path if hasattr(file.path, 'path') else file.path
data = PDFMetadata()
if reader is None or reader.metadata is None:
logger.error(f'Cannot get title of {path} PyPDF2 is not installed.')
data.title = super().get_title(file)
return data
data.title = reader.metadata.title
data.author = reader.metadata.author
return data
@classmethod
def get_words(cls, reader: PdfReader, path: pathlib.Path) -> Generator[str, None, None]:
"""
Reads all text facing up in a PDF.
Note, may return more than can be stored in Postgres. See: truncate_generator_bytes
"""
if PdfReader is None:
logger.error(f'Cannot index {path} PyPDF2 is not installed.')
yield ''
return
for page in reader.pages:
# Get all text facing up.
text = page.extract_text(0)
# Postgres does not allow null characters.
text = text.replace('\x00', '\uFFFD').strip()
if text:
yield text

View file

@ -2,12 +2,14 @@ import asyncio
import datetime
import functools
import glob
import json
import os
import pathlib
import re
import subprocess
from itertools import zip_longest
from pathlib import Path
from typing import List, Tuple, Union, Dict
from typing import List, Tuple, Union, Dict, Generator
import cachetools.func
import psycopg2
@ -16,14 +18,16 @@ from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name, chunks_by_stem, \
partition, ordered_unique_list, chunks, cancelable_wrapper, get_relative_to_media_directory
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidFile
partition, cancelable_wrapper, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_after_refresh
from wrolpi.dates import now, from_timestamp
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
from wrolpi.events import Events
from wrolpi.files.models import File
from wrolpi.vars import PYTEST, FILE_REFRESH_CHUNK_SIZE
from wrolpi.files.models import FileGroup
from wrolpi.lang import ISO_639_CODES
from wrolpi.tags import TagFile, Tag
from wrolpi.vars import PYTEST
try:
import magic
@ -37,7 +41,7 @@ except ImportError:
logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete_file', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
'get_mimetype', 'split_file_name_words']
'get_mimetype', 'split_file_name_words', 'get_primary_file']
@cachetools.func.ttl_cache(10_000, 30.0)
@ -158,7 +162,7 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
return mimetype
@functools.lru_cache(maxsize=1000)
@functools.lru_cache(maxsize=10_000)
def get_mimetype(path: Path) -> str:
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
if magic is None:
@ -177,265 +181,307 @@ def get_mimetype(path: Path) -> str:
# Special suffixes within WROLPi.
SUFFIXES = {
'.info.json',
'.en.vtt',
'.en.srt',
'.readability.json',
'.readability.txt',
'.readability.html',
}
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
@functools.lru_cache()
def split_path_stem_and_suffix(path: Union[pathlib.Path, str]) -> Tuple[str, str]:
@functools.lru_cache(maxsize=10_000)
def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = False) -> Tuple[str, str]:
"""Get the path's stem and suffix.
This function handles WROLPi suffixes like .info.json."""
if isinstance(path, str):
path = pathlib.Path(path)
full = str(path) # May or may not be absolute.
suffix = next(filter(lambda i: full.endswith(i), SUFFIXES), path.suffix)
if suffix:
full_ = str(path) # May or may not be absolute.
suffix = next(filter(lambda i: full_.endswith(i), SUFFIXES), path.suffix)
if suffix and full:
return f'{path.parent}/{path.name[:-1 * len(suffix)]}', suffix
elif suffix:
return path.name[:-1 * len(suffix)], suffix
# Path has no suffix.
return path.name, ''
if full:
return f'{path.parent}/{path.name}', ''
else:
return path.name, ''
refresh_logger = logger.getChild('refresh')
@wrol_mode_check
async def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime):
"""Upsert, index, and model all provided paths."""
# Update the idempotency if any files already in the DB.
def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
"""This generates `FileGroup.files` from a list of files."""
files = list()
for file in group:
mimetype = get_mimetype(file)
stat = file.stat()
modification_datetime = str(from_timestamp(stat.st_mtime))
_, suffix = split_path_stem_and_suffix(file)
size = stat.st_size
files.append(dict(
path=str(file),
mimetype=mimetype,
modification_datetime=modification_datetime,
size=size,
suffix=suffix,
))
return files
def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]], pre_sorted: bool = False) -> pathlib.Path:
"""Given a list of files, return the file that we can model or index."""
from modules.archive.lib import is_singlefile_file
if len(files) == 0:
raise ValueError(f'Cannot find primary file without any files')
elif len(files) == 1:
# Only one file is always the primary.
return files[0]
file_mimetypes = [(i, get_mimetype(i)) for i in files]
# EPUB has high priority.
mimetypes = {i[1] for i in file_mimetypes}
has_epub = any(i.startswith('application/epub') for i in mimetypes)
for file, mimetype in file_mimetypes:
# These are the files that can be modeled.
if is_singlefile_file(file):
return file
if mimetype.startswith('video/'):
return file
if mimetype.startswith('application/epub'):
return file
if mimetype.startswith('application/pdf') and not has_epub:
return file
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
# Only return a MOBI if no EPUB is present.
return file
# Secondary file types.
for file, mimetype in file_mimetypes:
if mimetype.startswith('image/'):
return file
logger.debug(f'Cannot find primary file for group: {files}')
# Can't find a typical primary file, just use the alphanumeric first file.
files = sorted(files) if pre_sorted is False else files
file = files[0]
return file
def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
"""Insert/update all records of the provided files.
Any inserted files will be marked with `indexed=false`. Any existing files will only be updated if their
size/modification_datetime changed.
It is assumed all files exist."""
idempotency = str(idempotency)
for chunk in chunks_by_stem(files, 100):
# Group all files by their shared full-stem. `chunk_by_stem` already sorted the files.
grouped = group_files_by_stem(chunk, pre_sorted=True)
# Convert the groups into file_groups. Extract common information into `file_group.data`.
values = dict()
for group in grouped:
full_stem, _ = split_path_stem_and_suffix(group[0], full=True)
# The primary file is the video/SingleFile/epub, etc.
primary_path = get_primary_file(group, pre_sorted=True)
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)
# The group uses a common modification_datetime so the group will be re-indexed when any of it's files are
# modified.
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
size = sum(i.stat().st_size for i in group)
files = json.dumps(_paths_to_files_dict(group))
values[full_stem] = (str(primary_path), modification_datetime, mimetype, size, files)
values = [(stem, False, idempotency, *i) for stem, i in values.items()]
with get_db_curs(commit=True) as curs:
values = mogrify(curs, values)
stmt = f'''
INSERT INTO file_group
(full_stem, indexed, idempotency, primary_path, modification_datetime, mimetype, size, files)
VALUES {values}
ON CONFLICT (full_stem) DO UPDATE
SET
idempotency=EXCLUDED.idempotency,
modification_datetime=EXCLUDED.modification_datetime,
mimetype=EXCLUDED.mimetype,
size=EXCLUDED.size,
files=(
-- Do not overwrite files unless the files have changed and need to be re-indexed.
CASE
WHEN
file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
THEN file_group.files
ELSE EXCLUDED.files
END
),
-- Preseve TRUE `indexed` only if the files have not changed.
indexed=(
file_group.indexed = true
AND file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
)
RETURNING id, indexed
'''
curs.execute(stmt)
need_index = [i[0] for i in curs.fetchall() if i[1] is False]
if need_index:
logger.info(f'Invalidated indexes of {len(need_index)} file groups')
logger.debug(f'Invalidated indexes of file groups {need_index}')
async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetime.datetime = None):
"""Discover all files in the directories provided in paths, as well as all files in paths.
All records for files in `paths` that do not exist will be deleted.
Will refuse to refresh when the media directory is empty."""
try:
next(get_media_directory().iterdir())
except:
# We don't want to delete a bunch of files which would exist if the drive was mounted.
raise UnknownDirectory(f'Refusing to refresh because media directory is empty or does not exist.')
if not paths:
return
raise ValueError('Must provide some paths to refresh.')
for idx, chunk in enumerate(map(set, chunks_by_stem(paths, FILE_REFRESH_CHUNK_SIZE))):
with get_db_session(commit=True) as session:
existing_files = list(session.query(File).filter(File.path.in_(chunk)))
existing_paths = {i.path for i in existing_files}
# Add any new files into the database.
new_files = []
if new_paths := (chunk - existing_paths):
new_files = [File(path=i, idempotency=idempotency) for i in new_paths]
session.add_all(new_files)
# Apply models to all files. Many files will have no model.
all_files = existing_files + new_files
apply_modelers(all_files, session)
idempotency = idempotency or now()
# Update idempotency for existing files after modelers. Otherwise SQLAlchemy forgets.
for existing_file in existing_files:
existing_file.idempotency = idempotency
exists, deleted = partition(lambda i: i.exists(), paths)
if idx > 0:
parent = all_files[0].path.parent
refresh_logger.debug(f'Committed chunk of {len(chunk)} files. Refreshing near {parent}')
# DISCOVER all files, upsert their records.
# Sleep between chunks to catch cancel.
await asyncio.sleep(0)
async def _refresh_directory_files_recursively(directory: pathlib.Path, idempotency: datetime):
"""Refresh all files in a directory, recursively refresh all files in subdirectories."""
directories = [directory, ]
while directories:
directory = directories.pop(0)
try:
files, new_directories = get_files_and_directories(directory)
if exists:
# Recursively upsert all files that exist, and files in the existing directories.
directories, files = partition(lambda i: i.is_dir(), exists)
while directories:
directory = directories.pop(0)
new_files, new_directories = get_files_and_directories(directory)
directories.extend(new_directories)
if files:
refresh_logger.info(f'Refreshing {len(files)} files in {directory}')
await _refresh_files_list(files, idempotency)
except Exception as e:
refresh_logger.error(f'Failed to refresh files in {directory}', exc_info=e)
except asyncio.CancelledError:
refresh_logger.error(f'Refresh canceled during {directory}')
raise
files.extend(new_files)
if len(files) >= 100:
# Wait until there are enough files to perform the upsert.
_upsert_files(files, idempotency)
files = list()
# Sleep to catch cancel.
await asyncio.sleep(0)
if files:
# Not enough files for the chunks above, finish what is left.
_upsert_files(files, idempotency)
@cancelable_wrapper
async def apply_indexers():
"""Finds all Files that have not been indexed and indexes them."""
from wrolpi.files.models import File
from wrolpi.db import get_db_session, get_db_curs
with get_db_curs() as curs:
curs.execute('SELECT path FROM file WHERE indexed = false AND associated = false')
missing_index = [pathlib.Path(i[0]) for i in curs.fetchall()]
refresh_logger.info(f'Indexing {len(missing_index)} files')
# Indexing can be slow, commit when its been too long.
last_commit = now()
max_seconds_between_commits = 30
for chunk in chunks(missing_index, 100):
with get_db_session(commit=True) as session:
files = session.query(File).filter(File.path.in_(chunk))
for file in files:
try:
file.do_index()
except Exception as e:
refresh_logger.error(f'Failed to index {file=}', exc_info=e)
if PYTEST:
raise
if (now() - last_commit).total_seconds() > max_seconds_between_commits:
refresh_logger.debug('Committing because its been too long.')
session.commit()
last_commit = now()
# Sleep to catch cancel.
await asyncio.sleep(0)
last_commit = now()
refresh_logger.info(f'Indexed chunk of {len(chunk)} files')
@wrol_mode_check
async def refresh_files_list(paths: List[str], include_files_near: bool = True):
"""Refresh a list of files. Will not refresh directories or subdirectories.
Parameters:
paths: A list of files that need to be refreshed. If not absolute, they are assumed to be relative to the
media directory.
include_files_near: If true, will also refresh files that share the stem of the provided files.
"""
refresh_logger.info(f'Refreshing {len(paths)} files list')
media_directory = get_media_directory()
# Convert all paths to absolute paths in the media directory.
paths = [pathlib.Path(i) for i in paths]
paths = [media_directory / i if not i.is_absolute() else i for i in paths]
if not paths:
raise FileNotFoundError('No files to refresh')
if include_files_near:
new_paths = []
for path in paths:
new_paths.extend(glob_shared_stem(path))
paths.extend(new_paths)
paths, deleted_paths = partition(lambda i: i.exists(), paths)
idempotency = now()
await _refresh_files_list(paths, idempotency)
# Delete any files near the paths that no longer exist.
paths_stems = [split_path_stem_and_suffix(i)[0] for i in paths]
paths_stems = [str(media_directory / i) for i in paths_stems]
with get_db_curs(commit=True) as curs:
# Get all paths that match the stems of these refreshed files.
params = dict(paths_stems=paths_stems, idempotency=idempotency)
curs.execute('''
UPDATE file SET idempotency = %(idempotency)s
WHERE full_stem = ANY (%(paths_stems)s)
RETURNING path
''', params)
existing_paths = [pathlib.Path(i[0]) for i in curs.fetchall()]
# Delete any files which no longer exist.
deleted_paths.extend([i for i in existing_paths if not i.exists()])
deleted_paths = list(map(str, deleted_paths))
curs.execute('DELETE FROM file WHERE path = ANY(%(deleted_paths)s)', dict(deleted_paths=deleted_paths))
apply_after_refresh()
await apply_indexers()
refresh_logger.info('Done refreshing files list')
wheres = ''
if paths:
# Use LIKE to delete any children of directories that are deleted.
wheres = ' OR '.join([curs.mogrify('full_stem LIKE %s', (f'{i}/%',)).decode() for i in paths])
deleted_files = ''
if deleted:
# Delete any paths that do not exist.
deleted_files = ' OR '.join([curs.mogrify('full_stem = %s', (str(i),)).decode() for i in deleted])
idempotency = curs.mogrify('%s', (idempotency,)).decode()
wheres = f' ( (idempotency != {idempotency} OR idempotency is null) AND ({wheres}))' if wheres else ''
stmt = f'''
DELETE FROM file_group
WHERE
-- Delete all known-deleted files.
{deleted_files}
-- Delete any files in the refreshed paths that were not updated.
{" OR " + wheres if deleted_files else wheres}
'''
logger.debug(stmt)
curs.execute(stmt)
@limit_concurrent(1) # Only one refresh at a time.
@wrol_mode_check
@cancelable_wrapper
async def refresh_files():
async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = True):
"""Find, model, and index all files in the media directory."""
with flags.refreshing:
refresh_logger.warning('Refreshing all files')
Events.send_global_refresh_started()
if isinstance(paths, str):
paths = [pathlib.Path(paths), ]
if isinstance(paths, pathlib.Path):
paths = [paths, ]
# TODO remove this later when everyone has migrated their files.
from modules.archive.lib import migrate_archive_files
migrate_archive_files()
with flags.refreshing:
if not paths:
refresh_logger.warning('Refreshing all files')
else:
refresh_logger.warning(f'Refreshing {", ".join(list(map(str, paths)))}')
if send_events:
Events.send_global_refresh_started()
idempotency = now()
# Add all files in the media directory to the DB.
await _refresh_directory_files_recursively(get_media_directory(), idempotency)
paths = paths or [get_media_directory()]
await refresh_discover_paths(paths, idempotency)
if send_events:
Events.send_global_refresh_discovery_completed()
Events.send_global_refresh_modeling_completed()
# Model all files that have not been indexed.
await apply_modelers()
if send_events:
Events.send_global_refresh_modeling_completed()
# Remove any records where the file no longer exists.
with get_db_curs(commit=True) as curs:
curs.execute('DELETE FROM file WHERE idempotency < %s OR idempotency is null RETURNING path',
(idempotency,))
deleted = list(curs.fetchall())
logger.debug(f'{deleted=}')
logger.warning(f'Removed {len(deleted)} missing files')
Events.send_global_refresh_delete_completed()
apply_after_refresh()
# Index the rest of the files that were not indexed by modelers.
await apply_indexers()
Events.send_global_refresh_indexing_completed()
if send_events:
Events.send_global_refresh_indexing_completed()
Events.send_global_refresh_completed()
# Cleanup any outdated file data.
await apply_after_refresh()
if send_events:
Events.send_global_after_refresh_completed()
if send_events:
Events.send_global_refresh_completed()
refresh_logger.warning('Done refreshing Files')
flags.refresh_complete.set()
@limit_concurrent(1)
@wrol_mode_check
@cancelable_wrapper
async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str], send_events: bool = True):
"""Upsert and index all files within a directory (recursively).
async def apply_indexers():
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
from wrolpi.files.models import FileGroup
while True:
# Continually query for Files that have not been indexed.
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20)
file_groups: List[FileGroup] = list(file_groups)
Any records of the files that are no longer in the directory will be removed."""
if isinstance(directory, str):
directory = pathlib.Path(directory)
if directory.is_file():
raise ValueError(f'Cannot refresh files of a file: {directory=}')
processed = 0
for file_group in file_groups:
processed += 1
try:
file_group.do_index()
except Exception:
# Error has already been logged in .do_index.
if PYTEST:
raise
# Always mark the FileGroup as indexed. We won't try to index it again.
file_group.indexed = True
with flags.refreshing_directory:
relative_path = str(get_relative_to_media_directory(directory))
if send_events:
Events.send_directory_refresh_started(f'Refresh of {repr(relative_path)} has started.')
# Sleep to catch cancel.
await asyncio.sleep(0)
# All Files older than this will be removed.
idempotency = now()
refresh_logger.info(f'Recursively refreshing all files in {directory}')
await _refresh_directory_files_recursively(directory, idempotency)
# Remove any records where the file no longer exists.
with get_db_curs(commit=True) as curs:
refresh_logger.debug(f'Deleting files no longer in {directory}')
params = dict(
directory=f'{directory}/%', # trailing / is important!
idempotency=idempotency,
)
stmt = '''
DELETE FROM file
WHERE
(idempotency < %(idempotency)s OR idempotency IS NULL)
AND path LIKE %(directory)s'''
curs.execute(stmt, params)
apply_after_refresh()
await apply_indexers()
if send_events:
Events.send_directory_refresh_completed(f'Refresh of {repr(relative_path)} has completed.')
refresh_logger.info(f'Done refreshing files in {directory}')
if processed < 20:
# Processed less than the limit, don't do the next query.
break
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None) -> \
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tags: List[str] = None) -> \
Tuple[List[dict], int]:
"""Search the Files table.
"""Search the FileGroup table.
Order the returned Files by their rank if `search_str` is provided. Return all files if
`search_str` is empty.
@ -448,9 +494,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
model: Only return files that match this model.
"""
params = dict(offset=offset, limit=limit)
wheres = ['associated = false']
wheres = []
selects = []
order_by = '1 ASC'
joins = []
if search_str:
params['search_str'] = search_str
@ -474,25 +521,54 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
params['model'] = model
wheres.append('model = %(model)s')
if tags:
where_, params_, join_ = tag_names_to_clauses(tags)
wheres.append(where_)
params.update(params_)
joins.append(join_)
wheres = '\n AND '.join(wheres)
selects = f"{', '.join(selects)}, " if selects else ""
join = '\n'.join(joins)
stmt = f'''
SELECT path, {selects} COUNT(*) OVER() AS total
FROM file
SELECT fg.id, {selects} COUNT(*) OVER() AS total
FROM file_group fg
{join}
{f"WHERE {wheres}" if wheres else ""}
GROUP BY fg.id
ORDER BY {order_by}
OFFSET %(offset)s LIMIT %(limit)s
'''
logger.debug(stmt)
results, total = handle_search_results(stmt, params)
results, total = handle_file_group_search_results(stmt, params)
return results, total
def handle_search_results(statement: str, params):
def tag_names_to_clauses(tags: List[str]):
"""Create the SQL necessary to filter the `file_group` table by the provided Tag names."""
params = dict()
if not tags:
return '', params, ''
where_tags = []
for idx, tag_name in enumerate(tags):
where_tags.append(f't.name = %(tag_name{idx})s')
params[f'tag_name{idx}'] = tag_name
where_tags = ' OR '.join(where_tags)
wheres = f'({where_tags})'
join = '''LEFT JOIN tag_file tf ON tf.file_group_id = fg.id
LEFT JOIN tag t ON t.id = tf.tag_id'''
return wheres, params, join
def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List[dict], int]:
"""
Execute the provided SQL statement and fetch the Files returned.
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to search.
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search.
See: `search`
"""
@ -504,25 +580,30 @@ def handle_search_results(statement: str, params):
# No videos
return [], 0
total = results[0]['total'] if results else 0
ranked_paths = [pathlib.Path(i['path']) for i in results]
ordered_ids = [i['id'] for i in results]
try:
ranks = [i['ts_rank'] for i in results]
except KeyError:
# No `ts_rank`, probably not searching `file.textsearch`.
# No `ts_rank`, probably not searching `file_group.textsearch`.
ranks = []
with get_db_session() as session:
results = get_ranked_models(ranked_paths, File, session=session)
try:
results = assign_file_prefetched_models(results)
except Exception as e:
logger.error(f'Failed to assign file prefetched models', exc_info=e)
if PYTEST:
raise
results = [i.__json__() for i in results]
# Preserve the ts_ranks, if any.
for idx, rank in enumerate(ranks):
results[idx]['ts_rank'] = rank
from modules.videos.models import Video
results = session.query(FileGroup, Video) \
.filter(FileGroup.id.in_(ordered_ids)) \
.outerjoin(Video, Video.file_group_id == FileGroup.id)
# Order FileGroups by their location in ordered_ids.
file_groups: List[Tuple[FileGroup, Video]] = sorted(results, key=lambda i: ordered_ids.index(i[0].id))
results = list()
for rank, (file_group, video) in zip_longest(ranks, file_groups):
video: Video
if video:
results.append(video.__json__())
else:
results.append(file_group.__json__())
# Preserve the ts_ranks, if any.
if rank:
results[-1]['ts_rank'] = rank
return results, total
@ -544,50 +625,6 @@ def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]:
return paths
def assign_file_prefetched_models(files: List[File]) -> List[File]:
"""
Prefetch the sub-models (matching `File.model`) in bulk to avoid issuing a query for every File.
>>> f = File(path='something', model='video')
>>> v = Video(video_path='something')
>>> results = assign_file_prefetched_models([f])
>>> results[0]
File(path='something', model='video')
>>> results[0].prefetched_model == v
True
"""
if not files:
return files
files = files.copy()
files_index = {i.path: files.index(i) for i in files}
session = Session.object_session(files[0])
files_by_model = dict()
for file in files:
if not file.model:
continue
try:
files_by_model[file.model].append(file)
except KeyError:
files_by_model[file.model] = [file, ]
logger.debug(f'Prefetching file models: {list(files_by_model.keys())}')
table_map = {i: get_model_by_table_name(i) for i in {j.model for j in files}}
for model, model_files in files_by_model.items():
table = table_map[model]
try:
sub_models = table.find_by_paths([i.path for i in model_files], session)
for sub_model in sub_models:
file: File = files[files_index[sub_model.primary_path]]
file.prefetched_model = sub_model
except NotImplementedError:
# Model has not defined the necessary methods.
pass
return files
def get_matching_directories(path: Union[str, Path]) -> List[str]:
"""
Return a list of directory strings that start with the provided path. If the path is a directory, return it's
@ -618,7 +655,7 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
WHITESPACE = re.compile(r'[\s_]')
def split_file_name_words(name: str) -> List[str]:
def split_file_name_words(name: str) -> str:
"""Split words in a filename.
Words are assumed to be separated by underscore, dash, or space. Words with a dash are included as a group, and
@ -641,34 +678,83 @@ def split_file_name_words(name: str) -> List[str]:
if suffix:
words.append(suffix.lstrip('.'))
words = ordered_unique_list(words)
words = ' '.join(words)
return words
except Exception as e:
logger.error(f'Failed to split filename into words: {name}', exc_info=e)
return [name]
return name
def get_file_statistics():
with get_db_curs() as curs:
curs.execute('''
SELECT
COUNT(path) AS "total_count",
COUNT(path) FILTER (WHERE file.mimetype = 'application/pdf') AS "pdf_count",
COUNT(path) FILTER (WHERE file.mimetype = 'application/zip') AS "zip_count",
COUNT(path) FILTER (WHERE file.mimetype LIKE 'video/%') AS "video_count",
COUNT(path) FILTER (WHERE file.mimetype LIKE 'image/%' AND file.associated = FALSE) AS "image_count",
COUNT(path) FILTER (WHERE file.mimetype LIKE 'audio/%' AND file.associated = FALSE) AS "audio_count",
SUM(size)::BIGINT AS "total_size"
COUNT(id) AS "total_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype = 'application/x-mobipocket-ebook') AS "ebook_count",
SUM(size)::BIGINT AS "total_size",
(SELECT COUNT(*) FROM archive) AS archive_count
FROM
file
file_group
''')
statistics = dict(curs.fetchall()[0])
statistics['total_size'] = statistics['total_size'] or 0
curs.execute('SELECT COUNT(*) FROM archive')
statistics['archive_count'] = curs.fetchall()[0][0]
curs.execute('SELECT COUNT(*) FROM ebook')
statistics['ebook_count'] = curs.fetchall()[0][0]
return statistics
def group_files_by_stem(files: List[pathlib.Path], pre_sorted: bool = False) -> \
Generator[List[pathlib.Path], None, None]:
"""
Return lists of paths, each list contains only paths that share a common stem.
>>> a = ['foo.txt', 'foo.mp4', 'bar.txt']
>>> group_files_by_stem(list(map((pathlib.Path, a))))
# Generator(['foo.mp4', 'foo.txt'], ['bar.txt'])
@param files: A list of paths to be grouped. All paths must be in the same directory!
@param pre_sorted: This function requires the files to be sorted, it will sort them by default.
"""
files = sorted(files) if not pre_sorted else files
file = files.pop(0)
group = [file, ]
prev_stem, _ = split_path_stem_and_suffix(file)
for file in files:
stem, suffix = split_path_stem_and_suffix(file)
if stem == prev_stem:
group.append(file)
continue
# Stem has changed, group is finished.
yield group
group = [file, ]
prev_stem = stem
yield group
def _get_tag(file_group_id: int, tag_name: str, session: Session):
file_group: FileGroup = session.query(FileGroup).filter_by(id=file_group_id).one_or_none()
if not file_group:
raise UnknownFile(f'Cannot find FileGroup with id {file_group_id}')
tag: Tag = Tag.find_by_name(tag_name, session)
if not tag:
raise UnknownTag(f'Cannot find Tag with name {tag_name}')
return file_group, tag
@optional_session(commit=True)
def add_file_group_tag(file_group_id: int, tag_name: str, session: Session = None) -> TagFile:
file_group, tag = _get_tag(file_group_id, tag_name, session)
tag_file = file_group.add_tag(tag, session)
return tag_file
@optional_session(commit=True)
def remove_file_group_tag(file_group_id: int, tag_name: str, session: Session = None):
file_group, tag = _get_tag(file_group_id, tag_name, session)
file_group.remove_tag(tag, session)

View file

@ -1,36 +1,81 @@
import pathlib
from datetime import datetime
from typing import List, Type, Optional
from sqlalchemy import Integer, Column, String, Computed, Boolean
from sqlalchemy.orm import deferred, Session, validates
from sqlalchemy import Column, String, Computed, BigInteger, Boolean
from sqlalchemy import types
from sqlalchemy.orm import deferred, relationship, Session
from sqlalchemy.orm.collections import InstrumentedList
import wrolpi.files.indexers
from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector, logger, get_model_by_table_name, \
truncate_object_bytes
from wrolpi.dates import TZDateTime, from_timestamp, now
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
from wrolpi.db import optional_session
from wrolpi.files import indexers
from wrolpi.media_path import MediaPathType
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
from wrolpi.tags import Tag, TagFile
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
class File(ModelHelper, Base):
"""A representation of a file on disk.
def into_db(obj):
if isinstance(obj, datetime):
return strftime(obj)
if isinstance(obj, pathlib.Path):
return str(obj)
return obj
Can be searched using `textsearch` once the data has been populated by the respective Indexer.
def out_of_db(media_directory, obj):
if obj and isinstance(obj, str) and obj.startswith(media_directory):
return pathlib.Path(obj)
if obj and isinstance(obj, str) and obj[0].isdigit():
try:
return strptime_ms(obj)
except Exception:
# Wasn't a datetime after all.
pass
return obj
class FancyJSON(types.TypeDecorator):
"""Converts pathlib.Path to strings when moving into DB, and vice versa.
Converts datetime to ISO strings when moving into DB, and vice versa.
"""
__tablename__ = 'file'
path: pathlib.Path = Column(MediaPathType, primary_key=True)
impl = types.JSON
associated = Column(Boolean, default=False)
full_stem = Column(String) # f'{directory}/{stem}'
idempotency = Column(TZDateTime, default=lambda: now())
indexed = Column(Boolean, default=False)
def process_bind_param(self, value, dialect):
if value:
# datetime(2020, 1, 1, 0, 0, 0) <-> '2020-01-01T00:00:00'
value = recursive_map(value, into_db)
return value
def process_result_value(self, value, dialect) -> None:
if value:
media_directory = str(get_media_directory())
value = recursive_map(value, lambda i: out_of_db(media_directory, i))
return value
class FileGroup(ModelHelper, Base):
__tablename__ = 'file_group'
id: int = Column(BigInteger, primary_key=True)
data = Column(FancyJSON) # populated by the modeler
files = Column(FancyJSON, nullable=False) # populated during discovery
full_stem = Column(String, unique=True)
idempotency = Column(TZDateTime)
indexed = Column(Boolean, default=lambda: False, nullable=False)
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
model = Column(String) # Filled out by the modeler. Will be the table name (video, archive, etc.).
model = Column(String) # "video", "archive", "ebook", etc.
modification_datetime = Column(TZDateTime)
size = Column(Integer)
suffix = Column(String) # aka the extension.
title = Column(String) # Provided by the modeler, or it will be the entire file name (with suffix).
# the Path of the file that can be modeled or indexed.
primary_path = Column(MediaPathType, nullable=False, unique=True)
size = Column(BigInteger, default=lambda: 0)
title = Column(String)
tag_files: InstrumentedList = relationship('TagFile')
a_text = deferred(Column(String))
b_text = deferred(Column(String))
@ -44,121 +89,153 @@ class File(ModelHelper, Base):
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
''')))
prefetched_model = None # This may be filled with record matching `model`.
def __repr__(self):
path = str(self.path.relative_to(get_media_directory()))
return f'<File {path=} mime={self.mimetype} model={self.model}>'
def __json__(self) -> dict:
path = self.path.relative_to(get_media_directory())
d = dict(
associated=self.associated,
directory=self.path.parent,
full_stem=self.full_stem,
key=path, # React browser expects this.
mimetype=self.mimetype,
model=self.model,
modified=self.modification_datetime, # React browser expects this.
path=path,
size=self.size,
suffix=self.suffix,
title=self.title,
)
if self.model and self.prefetched_model:
d[self.model] = self.prefetched_model.__json__()
elif self.model:
model: ModelHelper = get_model_by_table_name(self.model)
session = Session.object_session(self)
instance = model.find_by_path(self.path, session)
if not instance:
logger.warning(f'Could not find instance of {model} for {self.path}')
return d
d[self.model] = instance.__json__()
m = f'model={self.model}' if self.model else f'mimetype={self.mimetype}'
return f'<FileGroup id={self.id} {m} primary_path={repr(str(self.primary_path))}>'
def __json__(self):
from wrolpi.files.lib import split_path_stem_and_suffix
_, suffix = split_path_stem_and_suffix(self.primary_path)
tags = sorted([i.tag.name for i in self.tag_files])
d = {
'data': self.data,
'directory': self.primary_path.parent,
'files': self.my_files(),
'full_stem': pathlib.Path(self.full_stem) if self.full_stem else None,
'id': self.id,
'mimetype': self.mimetype,
'model': self.model,
'modified': self.modification_datetime or None,
'name': self.primary_path.name,
'primary_path': self.primary_path,
'size': self.size,
'suffix': suffix,
'tags': tags,
'title': self.title,
'key': self.primary_path,
}
return d
def __eq__(self, other):
if isinstance(other, File):
return other.path == self.path
if isinstance(other, pathlib.Path):
return other == self.path
if isinstance(other, str):
return other == str(self.path)
return False
@property
def name(self) -> str:
if self.title:
return self.title
return self.primary_path.name
@optional_session
def add_tag(self, tag: Tag, session: Session = None) -> TagFile:
return tag.add_tag(self, session)
@optional_session
def remove_tag(self, tag: Tag, session: Session = None):
tag.remove_tag(self, session)
def append_files(self, *paths: pathlib.Path):
"""Add all `paths` to this FileGroup.files."""
from wrolpi.files.lib import get_mimetype
new_files = list(self.files) if self.files else list()
for file in paths:
new_files.append(dict(path=file, mimetype=get_mimetype(file)))
self.files = new_files
def my_files(self, *mimetypes: str) -> List[dict]:
"""Return all files related to this group that match any of the provided mimetypes.
>>> FileGroup().my_files()
>>> FileGroup().my_files('application/pdf')
>>> FileGroup().my_files('video/')
"""
files = self.files
if not files:
logger.error(f'{self} has no files!')
raise ValueError(f'{self} has no files!')
# Convert path strings to Paths.
for i in range(len(files)):
files[i]['path'] = pathlib.Path(files[i]['path'])
if mimetypes:
files = list(filter(lambda i: any(i['mimetype'].startswith(m) for m in mimetypes), files))
# Sort files to avoid random order.
return sorted(files, key=lambda i: i['path'])
def my_paths(self, *mimetypes: str) -> List[pathlib.Path]:
return [i['path'] for i in self.my_files(*mimetypes)]
def my_video_files(self):
"""Return all my Files that are videos."""
return self.my_files('video/')
def my_json_files(self):
"""Return all my Files that are JSON."""
return self.my_files('application/json')
def my_poster_files(self) -> List[dict]:
"""Return all my Files that are images."""
return self.my_files('image/')
def my_subtitle_files(self) -> List[dict]:
"""Return all my Files that have text/srt or text/vtt mimetype."""
return self.my_files('text/srt', 'text/vtt')
def my_text_files(self) -> List[dict]:
"""Return all my Files that have a text mimetype. But, do not include subtitle files."""
text_files = self.my_files('text/')
subtitle_paths = [i['path'] for i in self.my_subtitle_files()]
return [i for i in text_files if i['path'] not in subtitle_paths]
def my_ebook_files(self) -> List[dict]:
return self.my_files('application/epub', 'application/x-mobipocket-ebook')
def delete(self):
"""Delete this FileGroup record, and all of its files."""
for path in self.my_paths():
path.unlink()
session = Session.object_session(self)
if session:
session.delete(self)
@property
def indexer(self):
from wrolpi.files import lib
return wrolpi.files.indexers.find_indexer(self)
def do_index(self, force_index: bool = False):
"""Gather any missing information about this file. Index the contents of this file using an Indexer."""
def indexer(self) -> Type[indexers.Indexer]:
if not self.mimetype:
self.do_stats()
raise ValueError(f'Cannot find indexer because {self} does not have a mimetype!')
return indexers.find_indexer(self.mimetype)
def do_index(self):
"""Gather any missing information about this file group. Index the contents of this file using an Indexer."""
try:
if force_index or self.indexed is not True:
# Get the indexer on a separate line for debugging.
indexer = self.indexer
# Only read the contents of the file once.
start = now()
self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self)
if (total_seconds := (now() - start).total_seconds()) > 1:
logger.info(f'Indexing {self.path} took {total_seconds} seconds')
self.indexed = True
# Get the indexer on a separate line for debugging.
indexer = self.indexer
# Only read the contents of the file once.
start = now()
self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self.primary_path)
self.title = self.title or self.primary_path.name
if (total_seconds := (now() - start).total_seconds()) > 1:
logger.info(f'Indexing {self.primary_path} took {total_seconds} seconds')
except Exception as e:
logger.error(f'Failed to index {self.path}', exc_info=e)
logger.error(f'Failed to index {self}', exc_info=e)
if PYTEST:
raise
def do_stats(self) -> bool:
"""Assign the mimetype, title, size, modification_time of this file.
Returns True if the file has changed since last index. Change is detected by comparing old size, mimetype
modification datetime."""
from wrolpi.files.lib import split_path_stem_and_suffix, split_file_name_words, get_mimetype
old_mimetype = self.mimetype
old_size = self.size
old_modification_datetime = self.modification_datetime
self.mimetype = get_mimetype(self.path)
stem, self.suffix = split_path_stem_and_suffix(self.path)
self.full_stem = f'{self.path.parent}/{stem}'
self.title = self.title or self.path.name
stat = self.path.stat()
self.size = stat.st_size
self.modification_datetime = from_timestamp(stat.st_mtime)
changed = old_mimetype != self.mimetype \
or old_size != self.size \
or old_modification_datetime != self.modification_datetime
if changed:
self.indexed = False
# Use file stem as a_text. This may be overwritten by the correct indexer.
self.a_text = self.a_text or split_file_name_words(stem)
# Clear old indexes, this file has changed and must be re-indexed.
self.b_text = self.c_text = self.d_text = None
return changed
@validates('path')
def validate_path(self, key, value):
value = pathlib.Path(value) if not isinstance(value, pathlib.Path) else value
if not value.is_absolute():
raise ValueError(f'File path must always be absolute! {value}')
if not str(value).startswith(str(get_media_directory())):
raise ValueError(f'File path {value} not in {get_media_directory()}')
return value
@classmethod
def upsert(cls, path, session: Session) -> Base:
if file := session.query(File).filter_by(path=path).one_or_none():
return file
file = File(path=path)
session.add(file)
return file
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
from wrolpi.files.lib import get_primary_file, split_path_stem_and_suffix, get_mimetype
file_group = FileGroup()
file_group.append_files(*paths)
file_group.full_stem, _ = split_path_stem_and_suffix(paths[0], full=True)
file_group.primary_path = get_primary_file(paths)
file_group.mimetype = get_mimetype(file_group.primary_path)
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
file_group.size = sum(i.stat().st_size for i in paths)
session.add(file_group)
return file_group
@staticmethod
@optional_session
def find_by_path(path, session) -> Optional['FileGroup']:
file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
return file_group

View file

@ -1,11 +1,14 @@
import gc
import asyncio
import dataclasses
import pathlib
from typing import List, Dict, Optional
from typing import Generator
from sqlalchemy.orm import Session
from wrolpi.common import register_modeler, logger
from wrolpi.files.models import File
from wrolpi.common import register_modeler, logger, truncate_generator_bytes, truncate_object_bytes, \
split_lines_by_length
from wrolpi.db import get_db_session
from wrolpi.files.indexers import Indexer
from wrolpi.files.models import FileGroup
from wrolpi.vars import FILE_MAX_PDF_SIZE, PYTEST, FILE_MAX_TEXT_SIZE
try:
from PyPDF2 import PdfReader
@ -14,32 +17,126 @@ except ImportError:
logger = logger.getChild(__name__)
__all__ = ['get_pdf_metadata', 'get_words', 'pdf_modeler']
@dataclasses.dataclass
class PDFMetadata:
title: str = None
author: str = None
text: str = None
def create_index(path):
file_title = Indexer.get_title(path)
def get_pdf_title(path: pathlib.Path) -> Optional[str]:
reader = PdfReader(path)
if reader.metadata and reader.metadata.title:
title = reader.metadata.title.strip()
# TODO collecting titles causes a memory leak. Why?
gc.collect()
return title
data = get_pdf_metadata(reader, path)
if path.stat().st_size > FILE_MAX_PDF_SIZE:
logger.warning(f'PDF too large to fully index: {path}')
return data.title, data.author, file_title, None
words = ''
try:
# PDFs are complex, don't fail to create title index if text extraction fails.
words = '\n'.join(truncate_generator_bytes(get_words(reader, path), FILE_MAX_TEXT_SIZE))
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
words = split_lines_by_length(words)
except Exception as e:
logger.error(f'Failed to index {path}', exc_info=e)
if PYTEST:
raise
return data.title, data.author, file_title, words
def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata:
"""Extract title/author from the PDF metadata."""
data = PDFMetadata()
if reader is None or reader.metadata is None:
logger.error(f'Cannot get title of {path} PyPDF2 is not installed.')
data.title = Indexer.get_title(path)
return data
data.title = reader.metadata.title
data.author = reader.metadata.author
return data
def get_words(reader: PdfReader, path: pathlib.Path) -> Generator[str, None, None]:
"""
Reads all text facing up in a PDF.
Note, may return more than can be stored in Postgres. See: truncate_generator_bytes
"""
if PdfReader is None:
logger.error(f'Cannot index {path} PyPDF2 is not installed.')
yield ''
return
for page in reader.pages:
# Get all text facing up.
text = page.extract_text(0)
# Postgres does not allow null characters.
text = text.replace('\x00', '\uFFFD').strip()
if text:
yield text
# PDFs can be large, limit how many are modeled at once.
PDF_PROCESSING_LIMIT = 10
@register_modeler
def pdf_modeler(groups: Dict[str, List[File]], _: Session):
"""Reads the contents of a PDF and extracts the title, attaches the title to the File.
This modeler does not claim any groups passed to it. Does NOT mark the PDFs as indexed."""
async def pdf_modeler():
"""Queries for any PDF files that have not been indexed. Each PDF found will be indexed."""
if not PdfReader:
logger.warning(f'Cannot get title from PDF without PyPDF2')
logger.warning(f'Cannot index PDF without PyPDF2')
return
pdf_files = (file for group in groups.values() for file in group if
file.indexed is not True and file.mimetype == 'application/pdf')
while True:
# Continually query for PDFs that have not been indexed.
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup) \
.filter(
# Get all groups that contain a PDF that have not been indexed.
FileGroup.indexed != True,
FileGroup.mimetype == 'application/pdf',
).limit(PDF_PROCESSING_LIMIT)
for file in pdf_files:
path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
try:
if title := get_pdf_title(path):
file.title = title
except Exception as e:
logger.error(f'Unable to get title of {file.path}', exc_info=e)
processed = 0
for file_group in file_groups:
processed += 1
pdf_files = file_group.my_files('application/pdf')
if not pdf_files:
logger.error('Query returned a group without a PDF!')
continue
# Assume the first PDF is the only PDF.
pdf_file = pdf_files[0]['path']
try:
title, author, file_title, contents = create_index(pdf_file)
file_group.data = {'author': author}
file_group.title = file_group.a_text = title
file_group.b_text = author
file_group.c_text = file_title
file_group.d_text = contents
except Exception as e:
logger.error(f'Failed to index PDF {pdf_file}', exc_info=e)
if PYTEST:
raise
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
file_group.indexed = True
if processed:
logger.debug(f'pdf_modeler processed {processed} PDFs')
if processed < PDF_PROCESSING_LIMIT:
# Did not reach limit, do not query again.
break
# Sleep to catch cancel.
await asyncio.sleep(0)

View file

@ -19,17 +19,12 @@ class FilesSearchRequest:
offset: Optional[int] = 0
mimetypes: List[str] = field(default_factory=list)
model: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
@dataclass
class DirectoryRefreshRequest:
directory: str = None
@dataclass
class FilesRefreshListRequest:
files: List[str] = None
include_files_near: bool = True
class FilesRefreshRequest:
paths: Optional[List[str]] = None
@dataclass
@ -40,3 +35,9 @@ class DirectoriesRequest:
@dataclass
class DirectoriesResponse:
directories: List[str]
@dataclass
class TagFileGroupPost:
file_group_id: int
tag_name: str

View file

@ -1,15 +1,13 @@
import json
from http import HTTPStatus
from itertools import zip_longest
from unittest import mock
import pytest
from wrolpi.errors import API_ERRORS, WROLModeEnabled
from wrolpi.files.models import File
from wrolpi.test.common import assert_dict_contains
from wrolpi.vars import PROJECT_DIR
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.vars import PROJECT_DIR
def test_list_files_api(test_client, make_files_structure, test_directory):
@ -21,7 +19,7 @@ def test_list_files_api(test_client, make_files_structure, test_directory):
'empty directory/',
'videos/other video.mp4',
'videos/some video.mp4',
'lost+found/', # Should always be ignored.
'lost+found/', # Should always be ignored.
]
files = make_files_structure(files)
files[0].write_text('bar contents')
@ -154,19 +152,9 @@ def test_delete_wrol_mode(test_client, wrol_mode_fixture):
assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code']
def do_search(test_client, search_str, total, expected):
content = json.dumps({'search_str': search_str})
request, response = test_client.post('/api/files/search', content=content)
assert response.json['totals']['files'] == total
for file, expected in zip_longest(response.json['files'], expected):
assert_dict_contains(file, expected)
# FileBrowser in React requires a key. We use the path.
assert file['path'] == file['key']
def test_files_search(test_session, test_client, make_files_structure):
def test_files_search(test_session, test_client, make_files_structure, assert_files_search):
# You can search an empty directory.
do_search(test_client, 'nothing', 0, [])
assert_files_search('nothing', [])
# Create files in the temporary directory. Add some contents so the mimetype can be tested.
files = [
@ -185,65 +173,11 @@ def test_files_search(test_session, test_client, make_files_structure):
request, response = test_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)])
do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)])
do_search(test_client, 'baz', 2, [
dict(path='baz baz two.mp4', mimetype='video/mp4', size=1056318),
dict(path='baz.mp4', mimetype='video/mp4', size=1056318),
])
do_search(test_client, 'two', 1, [dict(path='baz baz two.mp4', mimetype='video/mp4', size=1056318)])
do_search(test_client, 'nothing', 0, [])
# No search string returns all Files.
do_search(test_client, None, 4, [
dict(path='archives/bar.txt'),
dict(path='baz baz two.mp4'),
dict(path='baz.mp4'),
dict(path='foo_is_the_name.txt'),
])
def test_associated_files(test_session, test_client, make_files_structure):
mp4, png, j, txt = make_files_structure([
'video.mp4',
'video.png',
'video.info.json',
'not a video.txt',
])
test_client.post('/api/files/refresh')
mp4_file = test_session.query(File).filter_by(path=mp4).one()
png_file = test_session.query(File).filter_by(path=png).one()
j_file = test_session.query(File).filter_by(path=j).one()
txt_file = test_session.query(File).filter_by(path=txt).one()
def check_files_search(content, expected):
request, response = test_client.post('/api/files/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [{'path': i['path']} for i in response.json['files']] == expected
check_files_search(
{'search_str': 'video'},
[
{'path': 'not a video.txt'},
{'path': 'video.info.json'},
{'path': 'video.mp4'},
{'path': 'video.png'},
]
)
mp4_file.associated = False # the modeled file
png_file.associated = True
j_file.associated = True
test_session.commit()
# Associated files are last.
check_files_search(
{'search_str': 'video'},
[
{'path': 'not a video.txt'},
{'path': 'video.mp4'},
]
)
assert_files_search('foo', [dict(full_stem='foo_is_the_name')])
assert_files_search('bar', [dict(full_stem='archives/bar')])
assert_files_search('baz', [dict(full_stem='baz baz two'), dict(full_stem='baz')])
assert_files_search('two', [dict(full_stem='baz baz two')])
assert_files_search('nothing', [])
def test_directory_search(test_client, make_files_structure):
@ -271,19 +205,33 @@ def test_directory_search(test_client, make_files_structure):
assert_directories('food', [])
def test_refresh_files_list(test_session, test_client, make_files_structure):
def test_refresh_files_list(test_session, test_client, make_files_structure, test_directory):
"""The user can request to refresh specific files."""
make_files_structure(['bar.txt', 'bar.mp4'])
# Video file near `bar.txt` can be ignored.
content = json.dumps({'files': ['bar.txt'], 'include_files_near': False})
request, response = test_client.post('/api/files/refresh/list', content=content)
# Only the single file that was refreshed is discovered.
content = json.dumps({'paths': ['bar.txt']})
request, response = test_client.post('/api/files/refresh', content=content)
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(File).count() == 1
assert test_session.query(FileGroup).count() == 1
group: FileGroup = test_session.query(FileGroup).one()
assert len(group.files) == 1
request, response = test_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
group: FileGroup = test_session.query(FileGroup).one()
assert len(group.files) == 2
def test_file_statistics(test_session, test_client, example_pdf, example_mobi, example_epub, video_file):
def test_file_statistics(test_session, test_client, test_directory, example_pdf, example_mobi, example_epub,
video_file):
"""A summary of File statistics can be fetched."""
# Give each file a unique stem.
video_file.rename(test_directory / 'video.mp4')
example_pdf.rename(test_directory / 'pdf.pdf')
example_mobi.rename(test_directory / 'mobi.mobi')
example_epub.rename(test_directory / 'epub.epub')
# Statistics can be fetched while empty.
request, response = test_client.get('/api/statistics')
assert response.status_code == HTTPStatus.OK
@ -311,7 +259,7 @@ def test_file_statistics(test_session, test_client, example_pdf, example_mobi, e
'ebook_count': 2,
'image_count': 0,
'pdf_count': 1,
'total_count': 5,
'total_count': 4,
'video_count': 1,
'zip_count': 0,
}

View file

@ -1,12 +1,12 @@
import json
import shutil
from http import HTTPStatus
from unittest import mock
import pytest
from wrolpi.files.ebooks import EBook, MOBI_MIMETYPE, EPUB_MIMETYPE
from wrolpi.files.ebooks import EBook, EPUB_MIMETYPE
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import FileGroup
from wrolpi.test.common import assert_dict_contains
@ -17,38 +17,53 @@ async def test_index(test_session, test_directory, example_epub, example_mobi):
Covers can be discovered."""
await refresh_files()
ebook_epub, ebook_mobi = test_session.query(EBook).order_by(EBook.ebook_path)
ebook_epub: EBook
ebook_mobi: EBook
ebook: EBook = test_session.query(EBook).one()
assert ebook_epub.title == 'WROLPi Test Book'
assert ebook_epub.ebook_file.mimetype.startswith(EPUB_MIMETYPE)
assert ebook_epub.creator == 'roland'
assert ebook_epub.size == ebook_epub.ebook_file.size
assert ebook_epub.cover_path and ebook_epub.cover_path.is_file()
# The cover is the WROLPi logo.
assert ebook_epub.cover_file.size == 297099
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.file_group.mimetype.startswith(EPUB_MIMETYPE)
assert ebook.creator == 'roland'
assert ebook.size == 292579
assert ebook.file_group.data == {'creator': 'roland', 'title': 'WROLPi Test Book',
'cover_path': test_directory / 'example.jpeg',
'ebook_path': test_directory / 'example.epub',
}
assert ebook_epub.ebook_file.a_text, 'Book title was not indexed'
assert ebook_epub.ebook_file.b_text, 'Book creator was not indexed'
assert ebook_epub.ebook_file.d_text, 'Book text was not indexed'
# Mobi is not fully supported, title is the file name. No creator or cover.
assert ebook_mobi.title == 'example'
assert ebook_mobi.ebook_file.mimetype == MOBI_MIMETYPE
assert not ebook_mobi.creator
assert not ebook_mobi.cover_path
assert ebook.file_group.a_text, 'Book title was not indexed'
assert ebook.file_group.b_text, 'Book creator was not indexed'
assert ebook.file_group.d_text, 'Book text was not indexed'
# Both ebooks are assumed to be the same book, but different formats.
assert (epubs := ebook.file_group.my_files('application/epub+zip')) and len(epubs) == 1 \
and epubs[0]['path'] == example_epub
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
and mobis[0]['path'] == example_mobi
# Cover was discovered.
assert len(ebook.file_group.my_poster_files()) == 1
# EPUB, MOBI, JPEG.
assert len(ebook.file_group.files) == 3
# Ebooks can be deleted during refresh.
example_mobi.unlink()
example_epub.unlink()
await refresh_files()
assert test_session.query(EBook).count() == 0
assert test_session.query(EBook).count() == 1
assert not ebook.file_group.my_files('application/epub+zip')
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
and mobis[0]['path'] == example_mobi
# Cover was discovered.
assert len(ebook.file_group.my_poster_files()) == 1
# MOBI, JPEG.
assert len(ebook.file_group.files) == 2
# Extract/index should only be done once.
with mock.patch('wrolpi.files.ebooks.extract_ebook_data') as mock_extract_ebook_data:
mock_extract_ebook_data.side_effect = Exception('extract_ebook_data should not be called again')
await refresh_files()
@pytest.mark.asyncio
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory):
cover_path = example_epub.with_suffix('.jpg')
cover_path.write_bytes(image_bytes_factory())
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
# Cover file near the eBook was discovered.
assert ebook.cover_path.read_bytes() == cover_path.read_bytes()
def test_search(test_session, test_client, example_epub):
@ -59,75 +74,59 @@ def test_search(test_session, test_client, example_epub):
assert test_session.query(EBook).count() == 1
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.creator == 'roland'
assert ebook.ebook_file.a_text == 'WROLPi Test Book', 'Book title was not updated'
assert ebook.ebook_file.d_text, 'Book was not indexed'
assert ebook.file_group.a_text == 'WROLPi Test Book', 'Book title was not updated'
assert ebook.file_group.d_text, 'Book was not indexed'
content = dict(mimetypes=['application/epub', 'application/x-mobipocket-ebook'])
request, response = test_client.post('/api/files/search', content=json.dumps(content))
assert response.status == HTTPStatus.OK
assert response.json
result = response.json['files'][0]
assert result['path'] == 'example.epub'
assert result['mimetype'] == 'application/epub+zip'
file_group = response.json['file_groups'][0]
epub_file = file_group['files'][0]
assert epub_file['path'] == 'example.epub' and epub_file['mimetype'] == 'application/epub+zip'
assert_dict_contains(
result['ebook'],
{'cover_path': 'example.jpeg', 'ebook_path': 'example.epub'},
file_group['data'],
{'cover_path': 'example.jpeg', 'ebook_path': 'example.epub', 'title': 'WROLPi Test Book', 'creator': 'roland'},
)
content = dict(mimetypes=['application/epub+zip', 'application/x-mobipocket-ebook'])
request, response = test_client.post('/api/files/search', content=json.dumps(content))
result = response.json['files'][0]
assert result['path'] == 'example.epub'
assert result['mimetype'] == 'application/epub+zip'
content = dict(mimetypes=[])
# No Mobi ebook.
content = dict(mimetypes=['application/x-mobipocket-ebook'])
request, response = test_client.post('/api/files/search', content=json.dumps(content))
assert response.status == HTTPStatus.OK
assert response.json
assert len(response.json['file_groups']) == 0
@pytest.mark.asyncio
async def test_discover_calibre_cover(test_session, test_directory, example_epub, image_file):
async def test_discover_calibre_cover(test_session, test_directory, example_epub, example_mobi, image_file):
"""Calibre puts a cover near an ebook file, test if it can be found."""
# Create two files in the
# Create a Calibre metadata file.
metadata = test_directory / 'metadata.opf'
metadata.touch()
# Create an alternate format next to the epub. This will be ignored because it is not yet supported.
mobi_path = example_epub.with_suffix('.mobi')
mobi_path.touch()
await refresh_files()
assert test_session.query(EBook).count() == 1
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
assert ebook.file_group.title == 'WROLPi Test Book'
# The cover in the ebook was extracted.
assert ebook.cover_file.size == 297099
assert ebook.cover_file['path'] == test_directory / 'example.jpeg'
assert (test_directory / 'example.jpeg').is_file()
# Delete the extracted cover, use the cover image from Calibre.
ebook.cover_path.unlink()
test_session.delete(ebook.cover_file)
(test_directory / 'example.jpeg').unlink()
cover_image = test_directory / 'cover.jpg'
shutil.move(image_file, cover_image)
# Reset files.
for file_group in test_session.query(FileGroup):
test_session.delete(file_group)
test_session.commit()
# Calibre cover image was discovered, no cover was generated.
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
assert ebook.cover_file.size == 641
# Create a file nearby, its no longer a Calibre directory.
mobi_path = mobi_path.rename(test_directory / 'other book.mobi')
await refresh_files()
assert ebook.title == 'WROLPi Test Book'
assert ebook.cover_file.size == 297099
mobi_path.unlink()
# Delete the discovered cover, cover should be generated.
cover_image.unlink()
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
assert ebook.cover_file.size == 297099
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.cover_file['path'] == test_directory / 'cover.jpg'

View file

@ -10,21 +10,17 @@ from uuid import uuid4
import mock
import pytest
from PIL import Image
from sqlalchemy.orm import Session
from modules import videos
from wrolpi.common import get_media_directory, timer
from wrolpi.errors import InvalidFile
from wrolpi.common import timer
from wrolpi.dates import now, from_timestamp
from wrolpi.errors import InvalidFile, UnknownDirectory
from wrolpi.files import lib, indexers
from wrolpi.files.models import File
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile
from wrolpi.vars import PROJECT_DIR
def assert_files(session: Session, expected):
files = {str(i.path.relative_to(get_media_directory())) for i in session.query(File).all()}
assert files == set(expected)
def test_delete_file(make_files_structure, test_directory):
"""
File in the media directory can be deleted.
@ -64,6 +60,8 @@ def test_delete_file(make_files_structure, test_directory):
('/absolute/foo-something.info.json', ('foo-something', '.info.json')),
('/absolute/foo', ('foo', '')),
('/absolute/foo.bar', ('foo', '.bar')),
('foo.en.srt', ('foo', '.en.srt')),
('foo.pl.srt', ('foo', '.pl.srt')),
]
)
def test_split_path_stem_and_suffix(path, expected):
@ -71,7 +69,7 @@ def test_split_path_stem_and_suffix(path, expected):
@pytest.mark.asyncio
async def test_refresh_files(test_session, make_files_structure, test_directory):
async def test_refresh_files(test_session, make_files_structure, assert_file_groups):
"""All files in the media directory should be found when calling `refresh_files`"""
files = make_files_structure([
'foo.txt',
@ -81,18 +79,188 @@ async def test_refresh_files(test_session, make_files_structure, test_directory)
foo, bar, baz = files
await lib.refresh_files()
assert_files(test_session, ['bar.txt', 'baz.txt', 'foo.txt'])
assert_file_groups([
{'full_stem': 'foo', 'indexed': True},
{'full_stem': 'bar', 'indexed': True},
{'full_stem': 'baz', 'indexed': True}])
baz.unlink()
await lib.refresh_files()
assert_files(test_session, ['bar.txt', 'foo.txt'])
assert_file_groups([{'full_stem': 'foo', 'indexed': True}, {'full_stem': 'bar', 'indexed': True}])
foo.unlink()
bar.unlink()
await lib.refresh_files()
assert_files(test_session, [])
assert_file_groups([{'full_stem': 'bar', 'indexed': True}])
@pytest.mark.asyncio
async def test_refresh_bogus_files(test_session, make_files_structure, test_directory, assert_file_groups,
insert_file_group):
"""Bogus files are removed during a refresh."""
make_files_structure(['does exist.txt'])
await lib.refresh_files()
insert_file_group([test_directory / 'does not exist.txt'])
test_session.commit()
# Bogus file was inserted.
assert_file_groups([
{'full_stem': 'does exist', 'indexed': True},
{'full_stem': 'does not exist', 'indexed': True},
])
await lib.refresh_files()
assert test_session.query(FileGroup).count() == 1, 'Bogus file was not removed'
assert_file_groups([{'full_stem': 'does exist', 'indexed': True}])
@pytest.mark.asyncio
async def test_refresh_empty_media_directory(test_session, test_directory):
"""refresh_paths will refuse to refresh with an empty media directory."""
with pytest.raises(UnknownDirectory):
await lib.refresh_files()
@pytest.mark.asyncio
async def test__upsert_files(test_session, make_files_structure, test_directory, assert_file_groups, video_file,
srt_file3):
bar, baz = make_files_structure({
'dir1/bar.txt': None,
'baz.txt': 'baz file',
})
video_file = video_file.rename(test_directory / 'video.mp4')
srt_file3 = srt_file3.rename(test_directory / 'video.en.srt')
video_stem = str(test_directory / video_file.stem)
foo_mtime = from_timestamp(srt_file3.stat().st_mtime)
# All files are found because they are in this refresh request, or in the `dir1` directory.
idempotency = now()
lib._upsert_files([video_file, srt_file3, bar, baz], idempotency)
assert_file_groups([
{'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': False,
'files': [
{'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
]},
{'primary_path': bar, 'idempotency': idempotency,
'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}]
},
{'primary_path': baz, 'idempotency': idempotency,
'files': [{'path': baz, 'size': 8, 'suffix': '.txt', 'mimetype': 'text/plain'}]
},
])
# Modified files should be re-indexed.
bar.touch()
baz.write_text('new baz')
# Simulate that the video was indexed, it should not be re-indexed because it was not modified.
test_session.query(FileGroup).filter_by(primary_path=video_file).one().indexed = True
test_session.commit()
assert_file_groups(
[{'full_stem': video_stem, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True}],
assert_count=False)
# Only modified files need to be re-indexed.
lib._upsert_files([video_file, srt_file3, bar, baz], idempotency)
assert_file_groups([
{'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True,
'files': [
{'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
]},
{'primary_path': bar, 'idempotency': idempotency,
'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}],
},
{'primary_path': baz, 'idempotency': idempotency,
'files': [{'path': baz, 'size': 7, 'suffix': '.txt', 'mimetype': 'text/plain'}],
},
])
# Deleting SRT removes it from the video.
srt_file3.unlink()
lib._upsert_files([video_file, bar, baz], idempotency)
video_file_group: FileGroup = test_session.query(FileGroup).filter_by(full_stem=str(test_directory / 'video')).one()
assert len(video_file_group.files) == 1, 'SRT file was not removed from files'
@pytest.mark.asyncio
async def test_refresh_discover_paths(test_session, make_files_structure, test_directory, assert_files,
assert_file_groups):
foo, bar, baz = make_files_structure(['dir1/foo.txt', 'dir1/bar.txt', 'baz.txt'])
dir1 = foo.parent
# `refresh_paths` only refreshes the file requested.
await lib.refresh_discover_paths([foo, ])
assert_files([
{'path': 'dir1/foo.txt'},
])
# `refresh_paths` refreshes recursively.
await lib.refresh_discover_paths([foo.parent, ])
assert_files([
{'path': 'dir1/foo.txt'},
{'path': 'dir1/bar.txt'},
])
# `refresh_paths` finally discovers the file at the top of the media directory.
await lib.refresh_discover_paths([test_directory, ])
assert_files([
{'path': 'dir1/foo.txt'},
{'path': 'dir1/bar.txt'},
{'path': 'baz.txt'},
])
# Records for deleted files are deleted. Request a refresh of `dir1` so we indirectly refresh `foo`.
foo.unlink()
await lib.refresh_discover_paths([dir1, ])
assert_files([
{'path': 'dir1/bar.txt'},
{'path': 'baz.txt'},
])
# Records for all children of a directory are deleted.
bar.unlink()
dir1.rmdir()
await lib.refresh_discover_paths([dir1, ])
assert_files([
{'path': 'baz.txt'},
])
@pytest.mark.asyncio
async def test_refresh_discover_paths_groups(test_session, make_files_structure, test_directory):
make_files_structure(['dir1/foo.mp4', 'dir1/foo.info.json', 'baz.txt'])
await lib.refresh_discover_paths([test_directory, ], now())
# Two "foo" files, one "baz" file.
assert test_session.query(FileGroup).count() == 2
baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
# "foo" files are related to the "foo" group.
assert sorted([str(i['path'].relative_to(test_directory)) for i in foo.my_files()]) == [
'dir1/foo.info.json', 'dir1/foo.mp4',
]
# "bar" file is the only file related to the "bar" group.
assert len(baz.my_files()) == 1
assert str(baz.my_files()[0]['path'].relative_to(test_directory)) == 'baz.txt'
assert baz.indexed is False
@pytest.mark.asyncio
async def test_file_group_tag(test_session, make_files_structure, test_directory, tag_factory):
"""A FileGroup can be tagged."""
make_files_structure(['foo.mp4'])
await lib.refresh_discover_paths([test_directory, ], now())
one = tag_factory()
foo: FileGroup = test_session.query(FileGroup).one()
foo.add_tag(one)
test_session.commit()
tag_file2: TagFile = test_session.query(TagFile).one()
assert tag_file2 and tag_file2.file_group == foo
@pytest.mark.asyncio
@ -102,8 +270,8 @@ async def test_refresh_a_text_no_indexer(test_session, make_files_structure):
await lib.refresh_files()
files = {i.a_text for i in test_session.query(File).all()}
assert files == {'{bar,bar-bar}', '{foo}'}
files = {i.a_text for i in test_session.query(FileGroup)}
assert files == {'bar bar bar-bar', 'foo'}
@pytest.mark.asyncio
@ -113,51 +281,11 @@ async def test_refresh_many_files(test_session, make_files_structure):
make_files_structure([f'{uuid4()}.txt' for _ in range(count)])
with timer('first refresh'):
await lib.refresh_files()
assert test_session.query(File).count() == count
assert test_session.query(FileGroup).count() == count
with timer('second refresh'):
await lib.refresh_files()
assert test_session.query(File).count() == count
@pytest.mark.asyncio
async def test_refresh_files_list(test_session, make_files_structure, test_directory):
make_files_structure(['foo.txt', 'bar.txt'])
# Only foo.txt should have been refreshed.
await lib.refresh_files_list(['foo.txt'])
assert_files(test_session, ['foo.txt'])
# Both files should be found
await lib.refresh_files_list(['bar.txt'])
assert_files(test_session, ['foo.txt', 'bar.txt'])
# Both files should be unchanged.
await lib.refresh_files_list(['foo.txt', 'bar.txt'])
assert_files(test_session, ['foo.txt', 'bar.txt'])
with pytest.raises(FileNotFoundError):
# Some files must be refreshed.
await lib.refresh_files_list([])
# Files that share the stem can also be refreshed.
(test_directory / 'foo.mp4').touch()
await lib.refresh_files_list(['foo.txt', 'bar.txt'])
assert_files(test_session, ['foo.txt', 'foo.mp4', 'bar.txt'])
# Files that share the stem can also be ignored.
(test_directory / 'bar.mp4').touch()
await lib.refresh_files_list(['bar.txt'], include_files_near=False)
assert_files(test_session, ['foo.txt', 'foo.mp4', 'bar.txt'])
# Deleted files are removed.
(test_directory / 'foo.mp4').unlink()
(test_directory / 'bar.txt').unlink()
await lib.refresh_files_list(['foo.txt', 'bar.txt'], include_files_near=False)
assert_files(test_session, ['foo.txt', ])
# bar.mp4 is discovered near non-existent bar.txt.
await lib.refresh_files_list(['foo.txt', 'bar.txt'])
assert_files(test_session, ['foo.txt', 'bar.mp4'])
assert test_session.query(FileGroup).count() == count
@pytest.mark.asyncio
@ -181,33 +309,9 @@ async def test_refresh_cancel(test_session, make_files_structure, test_directory
task = asyncio.create_task(lib.refresh_files())
await assert_cancel(task)
task = asyncio.create_task(lib.refresh_directory_files_recursively(test_directory))
await assert_cancel(task)
@pytest.mark.asyncio
async def test_refresh_files_in_directory(test_session, make_files_structure, test_directory):
"""A subdirectory can be refreshed, files above it can be ignored."""
ignored, foo, similar = make_files_structure([
'ignored.txt',
'subdir/foo.txt',
'subdir-similarly-named.mp4',
])
await lib.refresh_directory_files_recursively(test_directory / 'subdir')
assert_files(test_session, ['subdir/foo.txt'])
await lib.refresh_directory_files_recursively(test_directory)
assert_files(test_session, ['ignored.txt', 'subdir/foo.txt', 'subdir-similarly-named.mp4'])
# The similarly named file is not deleted when refreshing the directory which shares the name.
foo.unlink()
await lib.refresh_directory_files_recursively(test_directory / 'subdir')
assert_files(test_session, ['ignored.txt', 'subdir-similarly-named.mp4'])
@pytest.mark.asyncio
async def test_mime_type(test_session, make_files_structure, test_directory):
async def test_mime_type(test_session, make_files_structure, test_directory, assert_files):
"""Files module uses the `file` command to get the mimetype of each file."""
from PIL import Image
@ -222,12 +326,17 @@ async def test_mime_type(test_session, make_files_structure, test_directory):
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', baz)
await lib.refresh_files()
assert_files(test_session, ['dir/foo text.txt', 'dir/bar.jpeg', 'dir/baz.mp4', 'dir/empty'])
assert_files([
{'path': 'dir/foo text.txt'},
{'path': 'dir/bar.jpeg'},
{'path': 'dir/baz.mp4'},
{'path': 'dir/empty'},
])
foo = test_session.query(File).filter_by(path=f'{test_directory}/dir/foo text.txt').one()
bar = test_session.query(File).filter_by(path=f'{test_directory}/dir/bar.jpeg').one()
baz = test_session.query(File).filter_by(path=f'{test_directory}/dir/baz.mp4').one()
empty = test_session.query(File).filter_by(path=f'{test_directory}/dir/empty').one()
foo = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/foo text.txt').one()
bar = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/bar.jpeg').one()
baz = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/baz.mp4').one()
empty = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/empty').one()
assert foo.mimetype == 'text/plain'
assert bar.mimetype == 'image/jpeg'
@ -257,29 +366,19 @@ async def test_files_indexer(test_session, make_files_structure, test_directory)
# Enable slow feature for testing.
# TODO can this be sped up to always be included?
with mock.patch('modules.videos.EXTRACT_SUBTITLES', True):
with mock.patch('modules.videos.lib.EXTRACT_SUBTITLES', True):
await lib.refresh_files()
text_file, zip_file, image_file, unknown_file, info_json_file, video_file \
= test_session.query(File).order_by(File.path)
text_file, zip_file, image_file, unknown_file, video_file \
= test_session.query(FileGroup).order_by(FileGroup.full_stem)
# Indexers are detected correctly.
assert text_file.path.suffix == '.txt' and text_file.indexer == indexers.TextIndexer
assert zip_file.path.suffix == '.zip' and zip_file.indexer == indexers.ZipIndexer
assert image_file.path.suffix == '.jpeg' and image_file.indexer == indexers.DefaultIndexer
assert unknown_file.path.suffix == '' and unknown_file.indexer == indexers.DefaultIndexer
assert info_json_file.path.suffix == '.json' and info_json_file.indexer == indexers.DefaultIndexer
assert video_file.path.suffix == '.mp4' and video_file.indexer == videos.VideoIndexer
def assert_file_properties(file: File, suffix):
assert file.suffix == suffix
assert_file_properties(text_file, '.txt')
assert_file_properties(zip_file, '.zip')
assert_file_properties(image_file, '.jpeg')
assert_file_properties(unknown_file, '')
assert_file_properties(info_json_file, '.info.json')
assert_file_properties(video_file, '.mp4')
assert text_file.mimetype == 'text/plain' and text_file.indexer == indexers.TextIndexer
assert zip_file.mimetype == 'application/zip' and zip_file.indexer == indexers.ZipIndexer
assert image_file.mimetype == 'image/jpeg' and image_file.indexer == indexers.DefaultIndexer
assert unknown_file.mimetype == 'inode/x-empty' and unknown_file.indexer == indexers.DefaultIndexer
# Video are indexed by the modeler, not by an indexer.
assert video_file.mimetype == 'video/mp4' and video_file.indexer == indexers.DefaultIndexer
# File are indexed by their titles and contents.
files, total = lib.search_files('file', 10, 0)
@ -289,15 +388,15 @@ async def test_files_indexer(test_session, make_files_structure, test_directory)
files, total = lib.search_files('contents', 10, 0)
assert total == 1 and files[0]['title'] == 'a text file.txt', 'The text file contains "contents".'
files, total = lib.search_files('video', 10, 0)
assert total == 1 and {i['title'] for i in files} == {'a video file.mp4'}, 'The video file contains "video".'
assert total == 1 and {i['title'] for i in files} == {'a video file'}, 'The video file contains "video".'
files, total = lib.search_files('yawn', 10, 0)
assert total == 1 and files[0]['title'] == 'a video file.mp4', 'The video file captions contain "yawn".'
assert total == 1 and files[0]['title'] == 'a video file', 'The video file captions contain "yawn".'
files, total = lib.search_files('bunny', 10, 0)
assert total == 1 and {i['title'] for i in files} == {'a zip file.zip'}, \
'The zip file contains a file with "bunny" in the title.'
with mock.patch('modules.videos.VideoIndexer.create_index') as mock_create_index:
mock_create_index.side_effect = Exception('This should not be called twice')
with mock.patch('modules.videos.models.Video.validate') as mock_validate:
mock_validate.side_effect = Exception('This should not be called twice')
await lib.refresh_files()
# Change the contents, the file should be re-indexed.
@ -308,11 +407,11 @@ async def test_files_indexer(test_session, make_files_structure, test_directory)
@pytest.mark.parametrize('name,expected', [
('this.txt', ['this', 'txt']),
('name', ['name']),
('name two', ['name', 'two']),
('this self-reliance_split.txt', ['this', 'self', 'reliance', 'self-reliance', 'split', 'txt']),
('-be_split!.txt', ['-be', 'split!', 'txt']),
('this.txt', 'this txt'),
('name', 'name'),
('name two', 'name two'),
('this self-reliance_split.txt', 'this self reliance self-reliance split txt'),
('-be_split!.txt', '-be split! txt'),
])
def test_split_file_name_words(name, expected):
assert lib.split_file_name_words(name) == expected
@ -327,11 +426,11 @@ async def test_large_text_indexer(test_session, make_files_structure):
'large_file.txt': 'foo ' * 1_000_000,
})
await lib.refresh_files()
assert test_session.query(File).count() == 1
assert test_session.query(FileGroup).count() == 1
assert large.is_file() and large.stat().st_size == 4_000_000
large_file: File = test_session.query(File).one()
large_file: FileGroup = test_session.query(FileGroup).one()
assert len(large_file.d_text) < large.stat().st_size
assert len(large_file.d_text) == 90_072
@ -392,53 +491,81 @@ def test_matching_directories(make_files_structure, test_directory):
assert matches == [str(test_directory / 'foo/qux')]
def test_pdf_indexer(example_pdf):
"""PDFs can be indexed by PDFIndexer."""
file = File(path=example_pdf)
a_text, b_text, c_text, d_text = indexers.PDFIndexer.create_index(file)
# The title extracted from the PDF.
assert a_text == 'WROLPi Test PDF'
# The author.
assert b_text == 'roland'
# The parsed file name.
assert c_text == ['example', 'pdf']
# All pages are extracted. Text is formatted to fit on vertical screen.
assert d_text == 'Page one\n' \
'Page two\n' \
'Lorem ipsum dolor sit amet,\n' \
'consectetur adipiscing elit, sed do\n' \
'eiusmod tempor incididunt ut labore et\n' \
'\n' \
'dolore magna aliqua. Ut enim ad minim\n' \
'veniam, quis nostrud exercitation\n' \
'ullamco laboris nisi ut \n' \
'aliquip ex ea commodo consequat. Duis\n' \
'aute irure dolor in reprehenderit in\n' \
'voluptate velit esse cillum \n' \
'dolore eu fugiat nulla pariatur.\n' \
'Excepteur sint occaecat cupidatat non\n' \
'proident, sunt in culpa qui officia \n' \
'deserunt mollit anim id est laborum.'
def test_pdf_indexer_max_size(example_pdf):
"""The contents of a large PDF are not indexed."""
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
file = File(path=example_pdf)
a_text, b_text, c_text, d_text = indexers.PDFIndexer.create_index(file)
# The file name is still indexed.
assert a_text == 'WROLPi Test PDF'
assert d_text is None
def test_get_mimetype(example_epub, example_mobi, example_pdf, image_file, video_file):
assert lib.get_mimetype(example_epub) == 'application/epub+zip'
assert lib.get_mimetype(example_mobi) == 'application/x-mobipocket-ebook'
assert lib.get_mimetype(example_pdf) == 'application/pdf'
assert lib.get_mimetype(image_file) == 'image/jpeg'
assert lib.get_mimetype(video_file) == 'video/mp4'
def test_group_files_by_stem(make_files_structure, test_directory):
make_files_structure([
'foo.mp4',
'foo.txt',
'foo.info.json',
'bar.txt',
'baz.txt',
])
files = list(test_directory.iterdir())
assert list(lib.group_files_by_stem(files)) == [
[test_directory / 'bar.txt'],
[test_directory / 'baz.txt'],
[test_directory / 'foo.info.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'],
]
def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, example_mobi, example_pdf,
singlefile_contents_factory):
"""Test that the most important file is returned from a list of files."""
srt_file3 = srt_file3.rename(test_directory / f'{video_file.stem}.srt')
# Same primary_file no matter the order.
assert lib.get_primary_file([video_file, srt_file3]) == video_file
assert lib.get_primary_file([srt_file3, video_file]) == video_file
singlefile = test_directory / 'singlefile.html'
singlefile.write_text(singlefile_contents_factory())
singlefile_text = test_directory / 'singlefile.txt'
singlefile_text.write_text('the contents')
assert lib.get_primary_file([singlefile, singlefile_text]) == singlefile
assert lib.get_primary_file([example_epub, example_mobi]) == example_epub
assert lib.get_primary_file([example_mobi, example_epub]) == example_epub
assert lib.get_primary_file([example_pdf, example_mobi]) == example_pdf
assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub
COMMON_JOIN = '''LEFT JOIN tag_file tf ON tf.file_group_id = fg.id
LEFT JOIN tag t ON t.id = tf.tag_id'''
@pytest.mark.parametrize(
'tags,wheres,params,join', [
(
['one'],
'(t.name = %(tag_name0)s)',
dict(tag_name0='one'),
COMMON_JOIN,
),
(
['one', 'two'],
'(t.name = %(tag_name0)s OR t.name = %(tag_name1)s)',
dict(tag_name0='one', tag_name1='two'),
COMMON_JOIN,
),
(
[],
'',
dict(),
'',
),
]
)
def test_tag_names_to_clauses(tags, wheres, params, join):
result = lib.tag_names_to_clauses(tags)
assert result[0] == wheres
assert result[1] == params
assert result[2] == join

View file

@ -2,18 +2,71 @@ from unittest import mock
import pytest
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import File
from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files import pdfs
from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_pdf_title(test_session, example_pdf):
"""PDF modeler extracts the PDF's title."""
await refresh_files()
"""PDF modeler extracts a PDF's title and contents."""
await files_lib.refresh_files()
with mock.patch('wrolpi.files.pdfs.get_pdf_title') as mock_get_pdf_title:
mock_get_pdf_title.side_effect = Exception('should not be read again')
await refresh_files()
with mock.patch('wrolpi.files.pdfs.create_index') as mock_create_index:
mock_create_index.side_effect = Exception('Should not index twice')
await files_lib.refresh_files()
pdf: File = test_session.query(File).one()
assert pdf.title == 'WROLPi Test PDF'
pdf: FileGroup = test_session.query(FileGroup).one()
assert pdf.indexed is True
assert pdf.title == pdf.a_text == 'WROLPi Test PDF'
assert pdf.b_text == 'roland'
assert pdf.c_text == 'example pdf'
assert pdf.d_text and pdf.d_text.startswith('Page one\n') and len(pdf.d_text) == 467
assert pdf.data and pdf.data.get('author') == 'roland'
@pytest.mark.asyncio
async def test_pdf_modeler(test_session, example_pdf):
"""PDFs can be indexed by PDFIndexer."""
files_lib._upsert_files([example_pdf], now())
await pdfs.pdf_modeler()
file_group = test_session.query(FileGroup).one()
a_text, b_text, c_text, d_text = file_group.a_text, file_group.b_text, file_group.c_text, file_group.d_text
# The title extracted from the PDF.
assert a_text == 'WROLPi Test PDF'
# The author.
assert b_text == 'roland'
# The parsed file name.
assert c_text == 'example pdf'
# All pages are extracted. Text is formatted to fit on vertical screen.
assert d_text == 'Page one\n' \
'Page two\n' \
'Lorem ipsum dolor sit amet,\n' \
'consectetur adipiscing elit, sed do\n' \
'eiusmod tempor incididunt ut labore et\n' \
'\n' \
'dolore magna aliqua. Ut enim ad minim\n' \
'veniam, quis nostrud exercitation\n' \
'ullamco laboris nisi ut \n' \
'aliquip ex ea commodo consequat. Duis\n' \
'aute irure dolor in reprehenderit in\n' \
'voluptate velit esse cillum \n' \
'dolore eu fugiat nulla pariatur.\n' \
'Excepteur sint occaecat cupidatat non\n' \
'proident, sunt in culpa qui officia \n' \
'deserunt mollit anim id est laborum.'
@pytest.mark.asyncio
async def test_pdf_indexer_max_size(test_session, example_pdf):
"""The contents of a large PDF are not indexed."""
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
files_lib._upsert_files([example_pdf], now())
await pdfs.pdf_modeler()
file_group = test_session.query(FileGroup).one()
assert file_group.d_text is None

185
wrolpi/lang.py Normal file
View file

@ -0,0 +1,185 @@
ISO_639_CODES = [
'aa', # Afar
'ab', # Abkhazian
'ae', # Avestan
'af', # Afrikaans
'ak', # Akan
'am', # Amharic
'an', # Aragonese
'ar', # Arabic
'as', # Assamese
'av', # Avaric
'ay', # Aymara
'az', # Azerbaijani
'ba', # Bashkir
'be', # Belarusian
'bg', # Bulgarian
'bi', # Bislama
'bm', # Bambara
'bn', # Bengali
'bo', # Tibetan
'br', # Breton
'bs', # Bosnian
'ca', # Catalan, Valencian
'ce', # Chechen
'ch', # Chamorro
'co', # Corsican
'cr', # Cree
'cs', # Czech
'cu', # Church Slavonic, Old Slavonic, Old Church Slavonic
'cv', # Chuvash
'cy', # Welsh
'da', # Danish
'de', # German
'dv', # Divehi, Dhivehi, Maldivian
'dz', # Dzongkha
'ee', # Ewe
'el', # Greek, Modern (1453–)
'en', # English
'eo', # Esperanto
'es', # Spanish, Castilian
'et', # Estonian
'eu', # Basque
'fa', # Persian
'ff', # Fulah
'fi', # Finnish
'fj', # Fijian
'fo', # Faroese
'fr', # French
'fy', # Western Frisian
'ga', # Irish
'gd', # Gaelic, Scottish Gaelic
'gl', # Galician
'gn', # Guarani
'gu', # Gujarati
'gv', # Manx
'ha', # Hausa
'he', # Hebrew
'hi', # Hindi
'ho', # Hiri Motu
'hr', # Croatian
'ht', # Haitian, Haitian Creole
'hu', # Hungarian
'hy', # Armenian
'hz', # Herero
'ia', # Interlingua (International Auxiliary Language Association)
'id', # Indonesian
'ie', # Interlingue, Occidental
'ig', # Igbo
'ii', # Sichuan Yi, Nuosu
'ik', # Inupiaq
'io', # Ido
'is', # Icelandic
'it', # Italian
'iu', # Inuktitut
'ja', # Japanese
'jv', # Javanese
'ka', # Georgian
'kg', # Kongo
'ki', # Kikuyu, Gikuyu
'kj', # Kuanyama, Kwanyama
'kk', # Kazakh
'kl', # Kalaallisut, Greenlandic
'km', # Central Khmer
'kn', # Kannada
'ko', # Korean
'kr', # Kanuri
'ks', # Kashmiri
'ku', # Kurdish
'kv', # Komi
'kw', # Cornish
'ky', # Kirghiz, Kyrgyz
'la', # Latin
'lb', # Luxembourgish, Letzeburgesch
'lg', # Ganda
'li', # Limburgan, Limburger, Limburgish
'ln', # Lingala
'lo', # Lao
'lt', # Lithuanian
'lu', # Luba-Katanga
'lv', # Latvian
'mg', # Malagasy
'mh', # Marshallese
'mi', # Maori
'mk', # Macedonian
'ml', # Malayalam
'mn', # Mongolian
'mr', # Marathi
'ms', # Malay
'mt', # Maltese
'my', # Burmese
'na', # Nauru
'nb', # Norwegian Bokmål
'nd', # North Ndebele
'ne', # Nepali
'ng', # Ndonga
'nl', # Dutch, Flemish
'nn', # Norwegian Nynorsk
'no', # Norwegian
'nr', # South Ndebele
'nv', # Navajo, Navaho
'ny', # Chichewa, Chewa, Nyanja
'oc', # Occitan
'oj', # Ojibwa
'om', # Oromo
'or', # Oriya
'os', # Ossetian, Ossetic
'pa', # Punjabi, Panjabi
'pi', # Pali
'pl', # Polish
'ps', # Pashto, Pushto
'pt', # Portuguese
'qu', # Quechua
'rm', # Romansh
'rn', # Rundi
'ro', # Romanian, Moldavian, Moldovan
'ru', # Russian
'rw', # Kinyarwanda
'sa', # Sanskrit
'sc', # Sardinian
'sd', # Sindhi
'se', # Northern Sami
'sg', # Sango
'si', # Sinhala, Sinhalese
'sk', # Slovak
'sl', # Slovenian
'sm', # Samoan
'sn', # Shona
'so', # Somali
'sq', # Albanian
'sr', # Serbian
'ss', # Swati
'st', # Southern Sotho
'su', # Sundanese
'sv', # Swedish
'sw', # Swahili
'ta', # Tamil
'te', # Telugu
'tg', # Tajik
'th', # Thai
'ti', # Tigrinya
'tk', # Turkmen
'tl', # Tagalog
'tn', # Tswana
'to', # Tonga (Tonga Islands)
'tr', # Turkish
'ts', # Tsonga
'tt', # Tatar
'tw', # Twi
'ty', # Tahitian
'ug', # Uighur, Uyghur
'uk', # Ukrainian
'ur', # Urdu
'uz', # Uzbek
've', # Venda
'vi', # Vietnamese
'vo', # Volapük
'wa', # Walloon
'wo', # Wolof
'xh', # Xhosa
'yi', # Yiddish
'yo', # Yoruba
'za', # Zhuang, Chuang
'zh', # Chinese
'zu', # Zulu
]

View file

@ -15,6 +15,7 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi import admin, status, flags
from wrolpi import tags
from wrolpi.admin import HotspotStatus
from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics
@ -24,7 +25,7 @@ from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError
from wrolpi.events import get_events
from wrolpi.files.lib import get_file_statistics
from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse, \
EventsRequest
EventsRequest, NewTagRequest, DeleteTagRequest
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
truthy_arg
from wrolpi.version import __version__
@ -32,6 +33,7 @@ from wrolpi.version import __version__
logger = logger.getChild(__name__)
api_app = Sanic(name='api_app')
api_app.config.FALLBACK_ERROR_FORMAT = 'json'
api_bp = Blueprint('RootAPI', url_prefix='/api')
@ -377,6 +379,26 @@ async def feed(_: Request, query: EventsRequest):
return json_response(dict(events=events, now=start))
@api_bp.get('/tag')
async def get_tags_request(_: Request):
tags_ = tags.get_tags()
return json_response(dict(tags=tags_))
@api_bp.post('/tag/new')
@validate(NewTagRequest)
async def post_new_tag(_: Request, body: NewTagRequest):
tags.new_tag(body.name, body.color)
return response.empty(HTTPStatus.CREATED)
@api_bp.post('/tag/delete')
@validate(DeleteTagRequest)
async def delete_tag_request(_: Request, body: DeleteTagRequest):
tags.delete_tag(body.name)
return response.empty(HTTPStatus.NO_CONTENT)
class CustomJSONEncoder(json.JSONEncoder):
def default(self, obj):
@ -391,7 +413,8 @@ class CustomJSONEncoder(json.JSONEncoder):
else:
# A datetime with no timezone is UTC.
obj = obj.replace(tzinfo=timezone.utc)
return obj.isoformat()
obj = obj.isoformat()
return obj
elif isinstance(obj, date):
# API always returns dates in UTC.
obj = datetime(obj.year, obj.month, obj.day, tzinfo=timezone.utc)

View file

@ -64,3 +64,14 @@ class JSONErrorResponse:
@dataclass
class EventsRequest:
after: Optional[str] = None
@dataclass
class NewTagRequest:
name: str
color: str
@dataclass
class DeleteTagRequest:
name: str

174
wrolpi/tags.py Normal file
View file

@ -0,0 +1,174 @@
import contextlib
from typing import List
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
from sqlalchemy.orm import relationship, Session
from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory
from wrolpi.db import optional_session
from wrolpi.errors import UnknownTag, UsedTag
logger = logger.getChild(__name__)
class TagFile(ModelHelper, Base):
__tablename__ = 'tag_file'
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
tag = relationship('Tag', back_populates='tag_files')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), primary_key=True)
file_group = relationship('FileGroup', back_populates='tag_files')
class Tag(ModelHelper, Base):
__tablename__ = 'tag'
id = Column(Integer, primary_key=True)
name = Column(String, unique=True, nullable=False)
color = Column(String)
tag_files = relationship('TagFile', back_populates='tag', cascade='all')
def __repr__(self):
name = self.name
color = self.color
return f'<Tag {name=} {color=}>'
def __json__(self):
return dict(
id=self.id,
name=self.name,
color=self.color,
)
@optional_session
def add_tag(self, file_group, session: Session = None) -> TagFile:
"""Add a TagFile for the provided FileGroup and this Tag.
@warning: Commits the session to keep the config in sync."""
from wrolpi.files.models import FileGroup
if not isinstance(file_group, FileGroup):
raise ValueError('Cannot apply tag to non-FileGroup')
tag_file = TagFile(file_group_id=file_group.id, tag_id=self.id)
logger.info(f'Tagging {file_group} with {self}')
session.add(tag_file)
session.flush([tag_file])
get_tags_config().add_tag(tag_file)
session.commit()
return tag_file
@optional_session
def remove_tag(self, file_group, session: Session = None):
"""Remove the record of a Tag applied to the FileGroup.
@warning: Commits the session to keep config in sync."""
from wrolpi.files.models import FileGroup
if not isinstance(file_group, FileGroup):
raise ValueError('Cannot remove tag of non-FileGroup')
tag_file = session.query(TagFile) \
.filter(TagFile.file_group_id == file_group.id, TagFile.tag_id == self.id) \
.one_or_none()
if tag_file:
session.delete(tag_file)
get_tags_config().remove_tag(tag_file)
session.commit()
else:
logger.warning(f'Could not find tag_file for FileGroup.id={file_group.id}/Tag.id={self.id=}')
@staticmethod
@optional_session
def find_by_name(name: str, session: Session) -> 'Tag':
tag = session.query(Tag).filter_by(name=name).one_or_none()
return tag
class TagsConfig(ConfigFile):
file_name = 'tags.yaml'
default_config = dict(
tags=list(),
)
@property
def tags(self) -> list:
return self._config['tags']
@tags.setter
def tags(self, value):
self.update({'tags': value})
def add_tag(self, tag_file: TagFile):
from wrolpi.files.models import FileGroup
file_group: FileGroup = tag_file.file_group
primary_path = str(file_group.primary_path.relative_to(get_media_directory()))
tag_name = tag_file.tag.name
tags = self.tags.copy()
value = [primary_path, tag_name]
if value not in tags:
tags.append(value)
self.tags = tags
def remove_tag(self, tag_file: TagFile):
from wrolpi.files.models import FileGroup
file_group: FileGroup = tag_file.file_group
primary_path = str(file_group.primary_path.relative_to(get_media_directory()))
tag_name = tag_file.tag.name
tags = self.tags.copy()
for idx, value in enumerate(self.tags):
if value == [primary_path, tag_name]:
tags = tags.copy()
tags.pop(idx)
self.tags = tags
break
TAGS_CONFIG: TagsConfig = TagsConfig(global_=True)
TEST_TAGS_CONFIG: TAGS_CONFIG = None
def get_tags_config():
global TEST_TAGS_CONFIG
if isinstance(TEST_TAGS_CONFIG, ConfigFile):
return TEST_TAGS_CONFIG
global TAGS_CONFIG
return TAGS_CONFIG
@contextlib.contextmanager
def test_tags_config():
global TEST_TAGS_CONFIG
TEST_TAGS_CONFIG = TagsConfig()
yield
TEST_TAGS_CONFIG = None
@optional_session
def get_tags(session: Session) -> List[Tag]:
tags = list(session.query(Tag))
return tags
@optional_session
def new_tag(name: str, color: str, session: Session) -> Tag:
tag = Tag(name=name, color=color)
session.add(tag)
session.flush([tag])
session.commit()
return tag
@optional_session
def delete_tag(name: str, session: Session = None):
tag: Tag = Tag.find_by_name(name, session)
if not tag:
raise UnknownTag(f'Cannot find tag {name}')
if tag.tag_files:
count = len(tag.tag_files)
raise UsedTag(f'Cannot delete {name} it is used by {count} files!')
session.delete(tag)
session.commit()

View file

@ -415,8 +415,7 @@ def test_chunks_by_stem(test_directory, make_files_structure):
assert_chunks(1, files, [['1.mp4', '1.txt'], ['2.mp4', '2.png', '2.txt'], ['3.mp4']])
assert_chunks(3, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt'], ['3.mp4']])
assert_chunks(6, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt', '3.mp4']])
# List is not sorted because it is less than the size.
assert_chunks(20, files, [['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']])
assert_chunks(20, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt', '3.mp4']])
@pytest.mark.parametrize(

View file

@ -17,7 +17,7 @@ def assert_events(expected: List[Dict], after=None):
assert_dict_contains(event, expected)
def test_events_api_feed(test_session, test_client):
def test_events_api_feed(test_session, test_client, example_pdf):
"""Events can be gotten from the API."""
# refresh_files() creates a few events.
request, response = test_client.post('/api/files/refresh')
@ -29,9 +29,10 @@ def test_events_api_feed(test_session, test_client):
expected = [
dict(event='global_refresh_completed', subject='refresh'),
dict(event='global_after_refresh_completed', subject='refresh'),
dict(event='global_refresh_indexing_completed', subject='refresh'),
dict(event='global_refresh_delete_completed', subject='refresh'),
dict(event='global_refresh_modeling_completed', subject='refresh'),
dict(event='global_refresh_discovery_completed', subject='refresh'),
dict(event='global_refresh_started', subject='refresh'),
]
for expected, event in zip_longest(expected, result):

96
wrolpi/test/test_tags.py Normal file
View file

@ -0,0 +1,96 @@
import pytest
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile, get_tags_config
@pytest.mark.asyncio
async def test_tags_file_group_json(test_session, make_files_structure, tag_factory, example_pdf):
"""The tags of a file are returned in it's JSON."""
tag_one = tag_factory()
tag_two = tag_factory()
await files_lib.refresh_files()
file_group: FileGroup = test_session.query(FileGroup).one()
tag_one.add_tag(file_group)
assert file_group.__json__()['tags'] == ['one']
file_group.add_tag(tag_two)
assert file_group.__json__()['tags'] == ['one', 'two']
file_group.remove_tag(tag_one)
assert file_group.__json__()['tags'] == ['two']
@pytest.mark.asyncio
async def test_tags_file_group(test_session, make_files_structure, tag_factory):
"""A FileGroup can be tagged with multiple Tags."""
make_files_structure(['video.mp4', 'video.png'])
await files_lib.refresh_files()
video_group: FileGroup = test_session.query(FileGroup).one()
tag_one = tag_factory()
assert len(tag_one.tag_files) == 0
tag_one.add_tag(video_group)
test_session.commit()
assert len(tag_one.tag_files) == 1
assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ]
tag_two = tag_factory()
tag_two.add_tag(video_group)
test_session.commit()
assert len(tag_one.tag_files) == 1
assert len(tag_two.tag_files) == 1
# Video has both tags.
assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', 'two']
assert test_session.query(TagFile).count() == 2
# Deleting the tag deletes the tag_file.
test_session.delete(tag_one)
test_session.commit()
assert sorted([i.tag.name for i in video_group.tag_files]) == ['two', ]
assert test_session.query(TagFile).count() == 1
@pytest.mark.asyncio
async def test_tags_config_(test_session, test_directory, tag_factory, example_pdf, video_file, test_tags_config):
"""Test that the config is updated when a FileGroup is tagged."""
await files_lib.refresh_files()
pdf: FileGroup = FileGroup.find_by_path(example_pdf, test_session)
video: FileGroup = FileGroup.find_by_path(video_file, test_session)
tag1 = tag_factory()
tag2 = tag_factory()
video.add_tag(tag1)
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
# Tag PDF twice.
pdf.add_tag(tag1)
pdf.add_tag(tag2)
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) in test_tags_config.read_text()
# One TagFile still exists.
pdf.remove_tag(tag1)
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) in test_tags_config.read_text()
# PDF is no longer tagged.
pdf.remove_tag(tag2)
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
video.remove_tag(tag1)
assert str(video_file.relative_to(test_directory)) not in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
# No more tags.
tags_config = get_tags_config()
assert isinstance(tags_config.tags, list) and len(tags_config.tags) == 0
# Removing non-existent tag does not error.
video.remove_tag(tag1)

View file

@ -47,7 +47,7 @@ def truthy_arg(value: str) -> bool:
DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
if not MEDIA_DIRECTORY.is_dir() and not PYTEST:
print(f'Media directory does not exist! {MEDIA_DIRECTORY}')
print(f'Media directory does not exist! {MEDIA_DIRECTORY}', file=sys.stderr)
CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config'
MODULES_DIR: Path = PROJECT_DIR / 'modules'