mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
Making example label a tag.
Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
parent
9fbfc8c0a7
commit
5615ce542d
71 changed files with 4160 additions and 3161 deletions
233
alembic/versions/3b6918aeca4b_file_group.py
Normal file
233
alembic/versions/3b6918aeca4b_file_group.py
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
"""File group.
|
||||
|
||||
Revision ID: 3b6918aeca4b
|
||||
Revises: 930b9b7adb79
|
||||
Create Date: 2023-02-27 14:52:46.943351
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '3b6918aeca4b'
|
||||
down_revision = '930b9b7adb79'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('DELETE FROM video') # noqa
|
||||
session.execute('ALTER TABLE video DROP COLUMN caption_path')
|
||||
session.execute('ALTER TABLE video DROP COLUMN censored')
|
||||
session.execute('ALTER TABLE video DROP COLUMN favorite')
|
||||
session.execute('ALTER TABLE video DROP COLUMN info_json_path')
|
||||
session.execute('ALTER TABLE video DROP COLUMN modification_datetime')
|
||||
session.execute('ALTER TABLE video DROP COLUMN poster_path')
|
||||
session.execute('ALTER TABLE video DROP COLUMN size')
|
||||
session.execute('ALTER TABLE video DROP COLUMN title')
|
||||
session.execute('ALTER TABLE video DROP COLUMN validated')
|
||||
session.execute('ALTER TABLE video DROP COLUMN video_path')
|
||||
|
||||
session.execute('DELETE FROM archive') # noqa
|
||||
session.execute('ALTER TABLE archive DROP COLUMN readability_json_path')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN readability_path')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN readability_txt_path')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN screenshot_path')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN singlefile_path')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN title')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN validated')
|
||||
|
||||
session.execute('DELETE FROM ebook') # noqa
|
||||
session.execute('ALTER TABLE ebook DROP COLUMN cover_path')
|
||||
session.execute('ALTER TABLE ebook DROP COLUMN ebook_path')
|
||||
session.execute('ALTER TABLE ebook DROP COLUMN title')
|
||||
|
||||
session.execute('DROP TABLE IF EXISTS file')
|
||||
|
||||
session.execute('''
|
||||
CREATE TABLE file_group (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
data JSON,
|
||||
files JSON,
|
||||
full_stem TEXT NOT NULL UNIQUE,
|
||||
idempotency TIMESTAMP WITH TIME ZONE,
|
||||
indexed BOOLEAN DEFAULT FALSE,
|
||||
mimetype TEXT NOT NULL,
|
||||
model TEXT,
|
||||
modification_datetime TIMESTAMP WITH TIME ZONE,
|
||||
primary_path TEXT NOT NULL UNIQUE,
|
||||
size BIGINT,
|
||||
title TEXT,
|
||||
a_text TEXT,
|
||||
b_text TEXT,
|
||||
c_text TEXT,
|
||||
d_text TEXT,
|
||||
textsearch tsvector GENERATED ALWAYS AS (
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
|
||||
) STORED
|
||||
)''')
|
||||
|
||||
session.execute('ALTER TABLE video ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
|
||||
session.execute('ALTER TABLE ebook ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
|
||||
session.execute('ALTER TABLE archive ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
|
||||
|
||||
session.execute('''
|
||||
CREATE TABLE tag (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
color VARCHAR(7)
|
||||
)''')
|
||||
|
||||
session.execute('''
|
||||
CREATE TABLE tag_file (
|
||||
tag_id INTEGER REFERENCES tag(id),
|
||||
file_group_id BIGINT REFERENCES file_group(id),
|
||||
created_at TIMESTAMP WITH TIME ZONE,
|
||||
UNIQUE (tag_id, file_group_id)
|
||||
)''')
|
||||
|
||||
session.execute('''
|
||||
INSERT INTO tag (name, color) VALUES
|
||||
('Automotive','#000000'),
|
||||
('Clothing','#9858C5'),
|
||||
('Comms','#ECD71C'),
|
||||
('Cooking','#00A04A'),
|
||||
('Electronics','#000000'),
|
||||
('Favorite','#000000'),
|
||||
('Firearms','#000000'),
|
||||
('Fire Safety','#000000'),
|
||||
('First Aid','#C40405'),
|
||||
('Food','#2F8055'),
|
||||
('Fuel','#000000'),
|
||||
('Garden','#3E8B22'),
|
||||
('Husbandry','#000000'),
|
||||
('Hygiene','#E6E6E6'),
|
||||
('Knots','#000000'),
|
||||
('Light','#EEEEEE'),
|
||||
('Maps','#333333'),
|
||||
('Mechanical','#7A7A7A'),
|
||||
('Medical','#000000'),
|
||||
('Nuclear','#000000'),
|
||||
('Power','#D837FF'),
|
||||
('Radio','#000000'),
|
||||
('Security','#333333'),
|
||||
('Shelter','#A18A62'),
|
||||
('Survival','#A18A62'),
|
||||
('Tools','#333333'),
|
||||
('Water','#291BC2')
|
||||
''')
|
||||
|
||||
session.execute('UPDATE wrolpi_flag SET refresh_complete = false')
|
||||
|
||||
if not DOCKERIZED:
|
||||
session.execute('ALTER TABLE public.file_group OWNER TO wrolpi')
|
||||
session.execute('ALTER TABLE public.tag OWNER TO wrolpi')
|
||||
session.execute('ALTER TABLE public.tag_file OWNER TO wrolpi')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('DROP TABLE IF EXISTS tag_file')
|
||||
session.execute('DROP TABLE IF EXISTS tag')
|
||||
|
||||
session.execute('ALTER TABLE video DROP COLUMN file_group_id')
|
||||
session.execute('ALTER TABLE ebook DROP COLUMN file_group_id')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN file_group_id')
|
||||
|
||||
session.execute('DROP TABLE file_group')
|
||||
|
||||
session.execute('''
|
||||
create table file
|
||||
(
|
||||
path text not null primary key,
|
||||
idempotency timestamp with time zone,
|
||||
indexed boolean default false,
|
||||
mimetype text,
|
||||
suffix text,
|
||||
model text,
|
||||
associated boolean default false,
|
||||
modification_datetime timestamp with time zone,
|
||||
size bigint,
|
||||
title text,
|
||||
a_text text,
|
||||
b_text text,
|
||||
c_text text,
|
||||
d_text text,
|
||||
textsearch tsvector generated always as ((
|
||||
((setweight(to_tsvector('english'::regconfig, COALESCE(a_text, ''::text)), 'A'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, ''::text)), 'B'::"char")) ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, ''::text)), 'C'::"char")) ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, ''::text)), 'D'::"char"))) stored,
|
||||
full_stem text
|
||||
);
|
||||
|
||||
alter table file
|
||||
owner to postgres;
|
||||
|
||||
create index file_mimetype_idx
|
||||
on file (mimetype);
|
||||
|
||||
create index file_textsearch_idx
|
||||
on file using gin (textsearch);
|
||||
|
||||
create index file_modification_datetime_idx
|
||||
on file (modification_datetime);
|
||||
|
||||
create index file_size_idx
|
||||
on file (size);
|
||||
|
||||
create index file_model_idx
|
||||
on file (model);
|
||||
|
||||
create index file_idempotency_idx
|
||||
on file (idempotency);
|
||||
|
||||
create index file_suffix_idx
|
||||
on file (suffix);
|
||||
|
||||
create index file_full_stem_idx
|
||||
on file (full_stem);
|
||||
''')
|
||||
|
||||
session.execute('DELETE FROM video') # noqa
|
||||
session.execute('ALTER TABLE video ADD COLUMN caption_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE video ADD COLUMN censored BOOLEAN DEFAULT FALSE')
|
||||
session.execute('ALTER TABLE video ADD COLUMN favorite TIMESTAMPTZ')
|
||||
session.execute('ALTER TABLE video ADD COLUMN info_json_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE video ADD COLUMN poster_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE video ADD COLUMN video_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE video ADD COLUMN modification_datetime TIMESTAMPTZ')
|
||||
session.execute('ALTER TABLE video ADD COLUMN size BIGINT')
|
||||
session.execute('ALTER TABLE video ADD COLUMN title TEXT')
|
||||
session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE')
|
||||
|
||||
session.execute('DELETE FROM archive') # noqa
|
||||
session.execute('ALTER TABLE archive ADD COLUMN readability_json_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE archive ADD COLUMN readability_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE archive ADD COLUMN readability_txt_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE archive ADD COLUMN screenshot_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE archive ADD COLUMN singlefile_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE archive ADD COLUMN title TEXT')
|
||||
session.execute('ALTER TABLE archive ADD COLUMN validated BOOLEAN DEFAULT FALSE')
|
||||
|
||||
session.execute('DELETE FROM ebook') # noqa
|
||||
session.execute('ALTER TABLE ebook ADD COLUMN cover_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE ebook ADD COLUMN ebook_path TEXT REFERENCES file(path)')
|
||||
session.execute('ALTER TABLE ebook ADD COLUMN title TEXT')
|
||||
|
||||
session.execute('UPDATE wrolpi_flag SET refresh_complete = false')
|
||||
|
||||
if not DOCKERIZED:
|
||||
session.execute('ALTER TABLE public.file OWNER TO wrolpi')
|
||||
16
app/package-lock.json
generated
16
app/package-lock.json
generated
|
|
@ -15,6 +15,7 @@
|
|||
"lodash": "^4.17.21",
|
||||
"query-string": "^8.1.0",
|
||||
"react": "^18.2.0",
|
||||
"react-colorful": "^5.6.1",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"react-qr-code": "^2.0.11",
|
||||
|
|
@ -14438,6 +14439,15 @@
|
|||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/react-colorful": {
|
||||
"version": "5.6.1",
|
||||
"resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz",
|
||||
"integrity": "sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==",
|
||||
"peerDependencies": {
|
||||
"react": ">=16.8.0",
|
||||
"react-dom": ">=16.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/react-dev-utils": {
|
||||
"version": "12.0.1",
|
||||
"resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-12.0.1.tgz",
|
||||
|
|
@ -27847,6 +27857,12 @@
|
|||
"whatwg-fetch": "^3.6.2"
|
||||
}
|
||||
},
|
||||
"react-colorful": {
|
||||
"version": "5.6.1",
|
||||
"resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz",
|
||||
"integrity": "sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==",
|
||||
"requires": {}
|
||||
},
|
||||
"react-dev-utils": {
|
||||
"version": "12.0.1",
|
||||
"resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-12.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
"lodash": "^4.17.21",
|
||||
"query-string": "^8.1.0",
|
||||
"react": "^18.2.0",
|
||||
"react-colorful": "^5.6.1",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"react-qr-code": "^2.0.11",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {Donate} from "./components/Donate";
|
|||
import {useEventsInterval} from "./Events";
|
||||
import {SemanticToastContainer} from "react-semantic-toasts-2";
|
||||
import {FilePreviewWrapper} from "./components/FilePreview";
|
||||
import {TagsContext, useTags} from "./Tags";
|
||||
|
||||
function PageNotFound() {
|
||||
const {t} = useContext(ThemeContext);
|
||||
|
|
@ -75,7 +76,8 @@ const router = createBrowserRouter(createRoutesFromElements(<Route
|
|||
</Route>));
|
||||
|
||||
export default function App() {
|
||||
const status = useStatusInterval();
|
||||
const statusValue = useStatusInterval();
|
||||
const tagsValue = useTags();
|
||||
useEventsInterval();
|
||||
|
||||
return <ThemeWrapper>
|
||||
|
|
@ -85,9 +87,11 @@ export default function App() {
|
|||
{/* Toasts can be on any page. */}
|
||||
<SemanticToastContainer position='top-right'/>
|
||||
<MediaContextProvider>
|
||||
<StatusContext.Provider value={status}>
|
||||
<RouterProvider router={router}/>
|
||||
</StatusContext.Provider>
|
||||
<TagsContext.Provider value={tagsValue}>
|
||||
<StatusContext.Provider value={statusValue}>
|
||||
<RouterProvider router={router}/>
|
||||
</StatusContext.Provider>
|
||||
</TagsContext.Provider>
|
||||
</MediaContextProvider>
|
||||
</FilePreviewWrapper>
|
||||
</ThemeWrapper>
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import {Divider, Icon, Message} from "semantic-ui-react";
|
|||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
import {refreshFiles} from "./api";
|
||||
import _ from "lodash";
|
||||
import {TagsSegment} from "./Tags";
|
||||
|
||||
function FlagsMessages({flags}) {
|
||||
if (!flags) {
|
||||
|
|
@ -62,7 +63,7 @@ function FlagsMessages({flags}) {
|
|||
export function Dashboard() {
|
||||
useTitle('Dashboard');
|
||||
|
||||
const {searchStr, setSearchStr} = useSearchFiles();
|
||||
const {searchStr, setSearchStr, activeTags} = useSearchFiles();
|
||||
|
||||
const {status} = useContext(StatusContext);
|
||||
const wrol_mode = status ? status['wrol_mode'] : null;
|
||||
|
|
@ -74,13 +75,14 @@ export function Dashboard() {
|
|||
|
||||
// Only show dashboard parts if not searching.
|
||||
let body;
|
||||
if (searchStr) {
|
||||
if (searchStr || activeTags) {
|
||||
body = <FilesSearchView showLimit={true} showSelectButton={true}/>;
|
||||
} else {
|
||||
body = <>
|
||||
{!wrol_mode && downloads}
|
||||
{/* Hide Status when user is starting a download */}
|
||||
{!downloadOpen && <DashboardStatus/>}
|
||||
<TagsSegment/>
|
||||
</>;
|
||||
}
|
||||
|
||||
|
|
|
|||
315
app/src/Tags.js
Normal file
315
app/src/Tags.js
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
import React, {useEffect, useState} from "react";
|
||||
import {addTag, deleteTag, getTags, postTag, removeTag} from "./api";
|
||||
import {
|
||||
Button as SButton,
|
||||
Confirm,
|
||||
Dimmer,
|
||||
Divider,
|
||||
Form,
|
||||
FormField,
|
||||
FormInput,
|
||||
Grid,
|
||||
Header,
|
||||
Label,
|
||||
Loader,
|
||||
Loader as SLoader,
|
||||
Modal,
|
||||
} from "semantic-ui-react";
|
||||
import {contrastingColor, HelpPopup} from "./components/Common";
|
||||
import {Button, Segment} from "./components/Theme";
|
||||
import _ from "lodash";
|
||||
import {Link} from "react-router-dom";
|
||||
import {HexColorPicker} from "react-colorful";
|
||||
|
||||
export const TagsContext = React.createContext({tags: [], fetchTags: null});
|
||||
|
||||
export function useTags() {
|
||||
const [tags, setTags] = React.useState(null);
|
||||
const [tagNames, setTagNames] = React.useState(null);
|
||||
|
||||
const fetchTags = async () => {
|
||||
try {
|
||||
const t = await getTags();
|
||||
setTags(t);
|
||||
setTagNames(t.map(i => i['name']));
|
||||
} catch (e) {
|
||||
setTags([]);
|
||||
setTagNames([]);
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
const NameToTagLabel = ({name, color, to, ...props}) => {
|
||||
if (tags && tags.length) {
|
||||
for (let i = 0; i < tags.length; i++) {
|
||||
const tag = tags[i];
|
||||
if (name === tag['name']) {
|
||||
const textColor = contrastingColor(tag['color']);
|
||||
const label = <Label
|
||||
size='large'
|
||||
style={{backgroundColor: tag['color'], color: textColor}}
|
||||
className='clickable'
|
||||
{...props} // onClick passed here.
|
||||
>
|
||||
{name}
|
||||
</Label>;
|
||||
return label;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Could not find tag by name, or no tags have been fetched.
|
||||
return <Label size='large'>{name}</Label>;
|
||||
}
|
||||
|
||||
const TagsGroup = ({tagNames, onClick}) => {
|
||||
if (!tagNames || tagNames.length === 0) {
|
||||
return <React.Fragment/>;
|
||||
}
|
||||
return <Label.Group tag>
|
||||
{tagNames.map(i => <NameToTagLabel key={i} name={i} onClick={() => onClick(i)}/>)}
|
||||
</Label.Group>
|
||||
}
|
||||
|
||||
const TagsLinkGroup = ({tagNames}) => {
|
||||
if (!tagNames || tagNames.length === 0) {
|
||||
return <React.Fragment/>;
|
||||
}
|
||||
return <Label.Group tag>
|
||||
{tagNames.map(i =>
|
||||
<Link key={i} to={`/?tag=${i}`} style={{margin: '0.3em'}}>
|
||||
<NameToTagLabel name={i}/>
|
||||
</Link>
|
||||
)}
|
||||
</Label.Group>
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
setTags([]);
|
||||
fetchTags();
|
||||
}, []);
|
||||
|
||||
return {tags, tagNames, NameToTagLabel, TagsGroup, TagsLinkGroup, fetchTags}
|
||||
}
|
||||
|
||||
function DeleteTagLabel({name, onConfirm}) {
|
||||
const {NameToTagLabel} = React.useContext(TagsContext);
|
||||
const [confirmOpen, setConfirmOpen] = useState(false);
|
||||
|
||||
return <div style={{marginTop: '1em'}}>
|
||||
<SButton color='red' onClick={() => setConfirmOpen(true)} icon='close'/>
|
||||
<Confirm
|
||||
id={`confirm${name}`}
|
||||
open={confirmOpen}
|
||||
content={`Are you sure you want to delete: ${name}?`}
|
||||
confirmButton='Delete'
|
||||
onCancel={() => setConfirmOpen(false)}
|
||||
onConfirm={async () => onConfirm(name)}
|
||||
/>
|
||||
<NameToTagLabel name={name}/>
|
||||
<br/>
|
||||
</div>
|
||||
}
|
||||
|
||||
function DeleteTagModal() {
|
||||
const {fetchTags, tagNames} = React.useContext(TagsContext);
|
||||
|
||||
const [open, setOpen] = useState(false);
|
||||
|
||||
const localDeleteTag = async (name) => {
|
||||
await deleteTag(name);
|
||||
if (fetchTags) {
|
||||
await fetchTags();
|
||||
}
|
||||
}
|
||||
|
||||
const content = tagNames && tagNames.length ?
|
||||
tagNames.map(i => <DeleteTagLabel key={i} name={i} onConfirm={localDeleteTag}/>)
|
||||
: <Dimmer><Loader inline active/></Dimmer>;
|
||||
|
||||
return <>
|
||||
<Modal closeIcon
|
||||
open={open}
|
||||
onOpen={() => setOpen(true)}
|
||||
onClose={() => setOpen(false)}
|
||||
>
|
||||
<Modal.Header>Delete Tags</Modal.Header>
|
||||
<Modal.Content>
|
||||
{content}
|
||||
</Modal.Content>
|
||||
</Modal>
|
||||
<SButton color='red' onClick={() => setOpen(true)}>
|
||||
Delete
|
||||
</SButton>
|
||||
</>
|
||||
}
|
||||
|
||||
function CreateTagModal() {
|
||||
const {fetchTags} = React.useContext(TagsContext);
|
||||
|
||||
const [open, setOpen] = useState(false);
|
||||
const [tagName, setTagName] = useState('');
|
||||
const [tagColor, setTagColor] = useState('#000000');
|
||||
const textColor = contrastingColor(tagColor);
|
||||
|
||||
const saveTag = async () => {
|
||||
await postTag(tagName, tagColor);
|
||||
if (fetchTags) {
|
||||
await fetchTags();
|
||||
}
|
||||
setTagName('');
|
||||
}
|
||||
|
||||
return <>
|
||||
<Modal closeIcon
|
||||
open={open}
|
||||
onOpen={() => setOpen(true)}
|
||||
onClose={() => setOpen(false)}
|
||||
>
|
||||
<Modal.Header>Create New Tag</Modal.Header>
|
||||
<Modal.Content>
|
||||
<Label.Group tag>
|
||||
<Label size='large' style={{backgroundColor: tagColor, color: textColor}}>
|
||||
{tagName || 'Example Tag'}
|
||||
</Label>
|
||||
</Label.Group>
|
||||
|
||||
<Divider/>
|
||||
|
||||
<Form autoComplete='off'>
|
||||
<FormField>
|
||||
<FormInput required
|
||||
label='Tag Name'
|
||||
type='text'
|
||||
placeholder='Unique Name'
|
||||
value={tagName}
|
||||
onChange={(e, {value}) => setTagName(value)}
|
||||
/>
|
||||
</FormField>
|
||||
|
||||
<HexColorPicker color={tagColor} onChange={setTagColor}/>
|
||||
|
||||
<SButton primary
|
||||
size='big'
|
||||
onClick={saveTag}
|
||||
style={{marginTop: '2em'}}
|
||||
disabled={!!!tagName}
|
||||
>
|
||||
Save
|
||||
</SButton>
|
||||
</Form>
|
||||
|
||||
</Modal.Content>
|
||||
</Modal>
|
||||
|
||||
<SButton primary onClick={() => setOpen(true)}>New</SButton>
|
||||
</>
|
||||
}
|
||||
|
||||
export function TagsModal({fileGroup, onClick}) {
|
||||
const {tags: usedTags} = fileGroup;
|
||||
const {TagsGroup} = useTags();
|
||||
const {tags: availableTags} = React.useContext(TagsContext);
|
||||
const [open, setOpen] = useState(false);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const localAddTag = async (name) => {
|
||||
setLoading(true);
|
||||
try {
|
||||
await addTag(fileGroup, name);
|
||||
await onClick();
|
||||
console.debug('Added tag');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const localRemoveTag = async (name) => {
|
||||
setLoading(true);
|
||||
try {
|
||||
await removeTag(fileGroup, name);
|
||||
await onClick();
|
||||
console.debug('Removed tag');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const availableTagNames = availableTags && availableTags.length ? availableTags.map(i => i['name']) : [];
|
||||
|
||||
const unusedTags = _.difference(availableTagNames, usedTags);
|
||||
|
||||
let availableTagsGroup = <SLoader active inline/>;
|
||||
if (unusedTags && unusedTags.length) {
|
||||
availableTagsGroup = <TagsGroup tagNames={unusedTags} onClick={localAddTag}/>;
|
||||
}
|
||||
|
||||
let usedTagsGroup = 'Add tags by clicking them below';
|
||||
if (usedTags && usedTags.length) {
|
||||
// TODO sort these labels.
|
||||
usedTagsGroup = <TagsGroup tagNames={usedTags} onClick={localRemoveTag}/>;
|
||||
}
|
||||
|
||||
const modal = <Modal closeIcon
|
||||
open={open}
|
||||
onOpen={() => setOpen(true)}
|
||||
onClose={() => setOpen(false)}>
|
||||
<Modal.Content>
|
||||
{loading && <Dimmer active><SLoader/></Dimmer>}
|
||||
<Header as='h4'>Used Tags</Header>
|
||||
|
||||
{usedTagsGroup}
|
||||
|
||||
<Divider/>
|
||||
|
||||
{availableTagsGroup}
|
||||
</Modal.Content>
|
||||
<Modal.Actions>
|
||||
<DeleteTagModal/>
|
||||
<CreateTagModal/>
|
||||
</Modal.Actions>
|
||||
</Modal>
|
||||
|
||||
return <>
|
||||
<Button icon='tag' onClick={() => setOpen(true)}/>
|
||||
{modal}
|
||||
</>
|
||||
}
|
||||
|
||||
export const taggedImageLabel = {corner: 'left', icon: 'tag', color: 'green'};
|
||||
|
||||
export const TagsDisplay = ({fileGroup, onClick}) => {
|
||||
const {TagsLinkGroup} = useTags();
|
||||
|
||||
return <Grid>
|
||||
<Grid.Row>
|
||||
<Grid.Column mobile={2} computer={1}>
|
||||
<TagsModal fileGroup={fileGroup} onClick={onClick}/>
|
||||
</Grid.Column>
|
||||
<Grid.Column mobile={13} computer={14}>
|
||||
<TagsLinkGroup tagNames={fileGroup['tags']}/>
|
||||
{_.isEmpty(fileGroup['tags']) &&
|
||||
<HelpPopup content='Click the button to add tags' iconSize={null}/>
|
||||
}
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
}
|
||||
|
||||
export const TagsSegment = () => {
|
||||
const {tagNames, TagsLinkGroup} = useTags();
|
||||
|
||||
let availableTagsGroup = <SLoader active inline/>;
|
||||
if (tagNames && tagNames.length) {
|
||||
availableTagsGroup = <TagsLinkGroup tagNames={tagNames}/>;
|
||||
}
|
||||
return <Segment>
|
||||
<Header as='h2'>Tags</Header>
|
||||
{availableTagsGroup}
|
||||
|
||||
<Divider/>
|
||||
|
||||
<DeleteTagModal/>
|
||||
<CreateTagModal/>
|
||||
</Segment>
|
||||
}
|
||||
116
app/src/api.js
116
app/src/api.js
|
|
@ -103,11 +103,11 @@ export async function getChannel(id) {
|
|||
return (await response.json())['channel'];
|
||||
}
|
||||
|
||||
export async function searchVideos(offset, limit, channelId, searchStr, order_by, filters = []) {
|
||||
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tags) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = parseInt(offset || 0);
|
||||
limit = parseInt(limit || DEFAULT_LIMIT);
|
||||
let body = {offset, limit, filters, order_by: order_by || 'rank'};
|
||||
let body = {offset, limit, order_by: order_by || 'rank'};
|
||||
|
||||
if (searchStr) {
|
||||
body['search_str'] = searchStr;
|
||||
|
|
@ -115,13 +115,16 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
|
|||
if (channelId) {
|
||||
body['channel_id'] = parseInt(channelId);
|
||||
}
|
||||
if (tags) {
|
||||
body['tag_names'] = tags.split(',');
|
||||
}
|
||||
|
||||
console.debug('searching videos', body);
|
||||
|
||||
let response = await apiPost(`${VIDEOS_API}/search`, body);
|
||||
if (response.status === 200) {
|
||||
let data = await response.json();
|
||||
return [data['files'], data['totals']['files']];
|
||||
return [data['file_groups'], data['totals']['file_groups']];
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
|
|
@ -136,7 +139,7 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
|
|||
export async function getVideo(video_id) {
|
||||
let response = await apiGet(`${VIDEOS_API}/video/${video_id}`);
|
||||
let data = await response.json();
|
||||
return [data['file'], data['prev'], data['next']];
|
||||
return [data['file_group'], data['prev'], data['next']];
|
||||
}
|
||||
|
||||
export async function deleteVideos(videoIds) {
|
||||
|
|
@ -203,12 +206,6 @@ export async function validateRegex(regex) {
|
|||
return (await response.json())['valid'];
|
||||
}
|
||||
|
||||
export async function favoriteVideo(video_id, favorite) {
|
||||
let body = {favorite: favorite, video_id};
|
||||
let response = await apiPost(`${VIDEOS_API}/favorite`, body);
|
||||
return (await response.json())['favorite'];
|
||||
}
|
||||
|
||||
export async function getVideosStatistics() {
|
||||
let response = await apiGet(`${VIDEOS_API}/statistics`);
|
||||
if (response.status === 200) {
|
||||
|
|
@ -353,7 +350,7 @@ export async function deleteArchives(archiveIds) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function searchArchives(offset, limit, domain, searchStr, order) {
|
||||
export async function searchArchives(offset, limit, domain, searchStr, order, tags) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = parseInt(offset || 0);
|
||||
limit = parseInt(limit || DEFAULT_LIMIT);
|
||||
|
|
@ -367,12 +364,15 @@ export async function searchArchives(offset, limit, domain, searchStr, order) {
|
|||
if (order) {
|
||||
body['order_by'] = order;
|
||||
}
|
||||
if (tags) {
|
||||
body['tag_names'] = tags.split(',');
|
||||
}
|
||||
|
||||
console.debug('searching archives', body);
|
||||
let response = await apiPost(`${ARCHIVES_API}/search`, body);
|
||||
if (response.status === 200) {
|
||||
let data = await response.json();
|
||||
return [data['archives'], data['totals']['archives']];
|
||||
return [data['file_groups'], data['totals']['file_groups']];
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
|
|
@ -380,6 +380,7 @@ export async function searchArchives(offset, limit, domain, searchStr, order) {
|
|||
description: 'Cannot search archives. See server logs.',
|
||||
time: 5000,
|
||||
});
|
||||
return [[], 0];
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -402,7 +403,7 @@ export async function getArchive(archiveId) {
|
|||
const response = await apiGet(`${ARCHIVES_API}/${archiveId}`);
|
||||
if (response.status === 200) {
|
||||
const data = await response.json();
|
||||
return [data['file'], data['history']];
|
||||
return [data['file_group'], data['history']];
|
||||
} else {
|
||||
toast({
|
||||
type: 'error', title: 'Archive Error', description: 'Unable to get Archive. See server logs.', time: 5000,
|
||||
|
|
@ -465,17 +466,7 @@ export async function deleteDownload(downloadId) {
|
|||
}
|
||||
}
|
||||
|
||||
const replaceFileDatetimes = (files) => {
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
let file = files[i];
|
||||
if (file['modified']) {
|
||||
files[i]['modified'] = file['modified'] * 1000;
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export async function filesSearch(offset, limit, searchStr, mimetypes, model) {
|
||||
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tags) {
|
||||
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
|
||||
if (mimetypes) {
|
||||
body['mimetypes'] = mimetypes;
|
||||
|
|
@ -483,14 +474,16 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model) {
|
|||
if (model) {
|
||||
body['model'] = model;
|
||||
}
|
||||
if (tags) {
|
||||
body['tag_names'] = tags.split(',');
|
||||
}
|
||||
console.debug('searching files', body);
|
||||
const response = await apiPost(`${API_URI}/files/search`, body);
|
||||
|
||||
if (response.status === 200) {
|
||||
let data = await response.json();
|
||||
let [files, total] = [data['files'], data['totals']['files']];
|
||||
files = replaceFileDatetimes(files);
|
||||
return [files, total];
|
||||
let [file_groups, total] = [data['file_groups'], data['totals']['file_groups']];
|
||||
return [file_groups, total];
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
|
|
@ -507,9 +500,9 @@ export async function refreshFiles() {
|
|||
}
|
||||
|
||||
export async function refreshDirectoryFiles(directory) {
|
||||
let body = {directory};
|
||||
let body = {paths: [directory]};
|
||||
try {
|
||||
return await apiPost(`${API_URI}/files/refresh/directory`, body);
|
||||
return await apiPost(`${API_URI}/files/refresh`, body);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
toast({
|
||||
|
|
@ -526,7 +519,6 @@ export async function getFiles(directories) {
|
|||
let body = {directories: directories || []};
|
||||
let response = await apiPost(`${API_URI}/files`, body);
|
||||
let {files} = await response.json();
|
||||
files = replaceFileDatetimes(files);
|
||||
return files;
|
||||
}
|
||||
|
||||
|
|
@ -676,3 +668,67 @@ export async function getEvents(after) {
|
|||
return await response.json();
|
||||
}
|
||||
}
|
||||
|
||||
export async function getTags() {
|
||||
let uri = `${API_URI}/tag`;
|
||||
let response = await apiGet(uri);
|
||||
if (response.status === 200) {
|
||||
const body = await response.json();
|
||||
return body['tags'];
|
||||
}
|
||||
}
|
||||
|
||||
export async function addTag(fileGroup, name) {
|
||||
const {id} = fileGroup;
|
||||
const uri = `${API_URI}/files/tag`;
|
||||
const body = {file_group_id: id, tag_name: name};
|
||||
let response = await apiPost(uri, body);
|
||||
if (response.status !== 201) {
|
||||
console.error('Failed to add tag');
|
||||
}
|
||||
}
|
||||
|
||||
export async function removeTag(fileGroup, name) {
|
||||
const {id} = fileGroup;
|
||||
const uri = `${API_URI}/files/untag`;
|
||||
const body = {file_group_id: id, tag_name: name};
|
||||
let response = await apiPost(uri, body)
|
||||
if (response.status !== 204) {
|
||||
console.error('Failed to remove tag');
|
||||
}
|
||||
}
|
||||
|
||||
export async function postTag(name, color) {
|
||||
const body = {name: name, color: color};
|
||||
const uri = `${API_URI}/tag/new`;
|
||||
let response = await apiPost(uri, body);
|
||||
if (response.status !== 201) {
|
||||
console.error('Failed to create new tag');
|
||||
toast({
|
||||
type: 'error', title: 'Error!', description: 'Unable to save tag', time: 5000,
|
||||
})
|
||||
} else {
|
||||
toast({
|
||||
type: 'info', title: 'Created new tag', description: `Created new tag: ${name}`, time: 2000,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteTag(name) {
|
||||
const body = {name: name};
|
||||
const uri = `${API_URI}/tag/delete`;
|
||||
let response = await apiPost(uri, body);
|
||||
if (response.status === 400) {
|
||||
const content = await response.json();
|
||||
if (content['code'] === 38) {
|
||||
toast({
|
||||
type: 'error', title: 'Error!', description: content['message'], time: 5000,
|
||||
})
|
||||
}
|
||||
} else if (response.status !== 204) {
|
||||
console.error('Failed to delete tag');
|
||||
toast({
|
||||
type: 'error', title: 'Error!', description: `Unable to delete tag: ${name}`, time: 5000,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,20 +34,21 @@ import {deleteArchives, postDownload} from "../api";
|
|||
import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {useArchive, useDomains, useQuery, useSearchArchives} from "../hooks/customHooks";
|
||||
import {FileCards, FilesView} from "./Files";
|
||||
import {FileCards, FileRowTagIcon, FilesView} from "./Files";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
import Dropdown from "semantic-ui-react/dist/commonjs/modules/Dropdown";
|
||||
import _ from "lodash";
|
||||
import {ThemeContext} from "../contexts/contexts";
|
||||
import {Button, Card, CardIcon, Header, Loader, Placeholder, Segment} from "./Theme";
|
||||
import {SortableTable} from "./SortableTable";
|
||||
import {taggedImageLabel, TagsDisplay} from "../Tags";
|
||||
|
||||
function ArchivePage() {
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
const [updateOpen, setUpdateOpen] = useState(false);
|
||||
const navigate = useNavigate();
|
||||
const {archiveId} = useParams();
|
||||
const {archiveFile, history} = useArchive(archiveId);
|
||||
const {archiveFile, history, fetchArchive} = useArchive(archiveId);
|
||||
|
||||
let title;
|
||||
if (archiveFile && archiveFile.archive) {
|
||||
|
|
@ -65,11 +66,11 @@ function ArchivePage() {
|
|||
</>
|
||||
}
|
||||
|
||||
const {archive} = archiveFile;
|
||||
const {data} = archiveFile;
|
||||
|
||||
const singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null;
|
||||
const screenshotUrl = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
|
||||
const readabilityUrl = archive.readability_path;
|
||||
const singlefileUrl = data.singlefile_path ? `/media/${encodeURIComponent(data.singlefile_path)}` : null;
|
||||
const screenshotUrl = data.screenshot_path ? `/media/${encodeURIComponent(data.screenshot_path)}` : null;
|
||||
const readabilityUrl = data.readability_path;
|
||||
|
||||
const singlefileButton = <ExternalCardLink to={singlefileUrl}>
|
||||
<Button content='View' color='blue'/>
|
||||
|
|
@ -85,12 +86,12 @@ function ArchivePage() {
|
|||
|
||||
const localDeleteArchive = async () => {
|
||||
setDeleteOpen(false);
|
||||
await deleteArchives([archive.id]);
|
||||
await deleteArchives([data.id]);
|
||||
navigate(-1);
|
||||
}
|
||||
const localUpdateArchive = async () => {
|
||||
setUpdateOpen(false);
|
||||
await postDownload(archive.url, 'archive');
|
||||
await postDownload(data.url, 'archive');
|
||||
}
|
||||
|
||||
const updateButton = (<>
|
||||
|
|
@ -121,7 +122,7 @@ function ArchivePage() {
|
|||
historyList = <FileCards files={history}/>;
|
||||
}
|
||||
|
||||
const domain = archive.domain ? archive.domain.domain : null;
|
||||
const domain = data.domain ? data.domain.domain : null;
|
||||
let domainHeader;
|
||||
if (domain) {
|
||||
const domainUrl = `/archive?domain=${domain}`;
|
||||
|
|
@ -133,10 +134,10 @@ function ArchivePage() {
|
|||
}
|
||||
|
||||
let urlHeader;
|
||||
if (archive.url) {
|
||||
if (data.url) {
|
||||
urlHeader = <Header as='h5'>
|
||||
<ExternalCardLink to={archive.url}>
|
||||
{textEllipsis(archive.url)}
|
||||
<ExternalCardLink to={data.url}>
|
||||
{textEllipsis(data.url)}
|
||||
</ExternalCardLink>
|
||||
</Header>;
|
||||
}
|
||||
|
|
@ -147,9 +148,9 @@ function ArchivePage() {
|
|||
<Segment>
|
||||
{screenshot}
|
||||
<ExternalCardLink to={singlefileUrl}>
|
||||
<Header as='h2'>{textEllipsis(archive.title || archive.url)}</Header>
|
||||
<Header as='h2'>{textEllipsis(data.title || data.url)}</Header>
|
||||
</ExternalCardLink>
|
||||
<Header as='h3'>{isoDatetimeToString(archive.archive_datetime)}</Header>
|
||||
<Header as='h3'>{isoDatetimeToString(data.archive_datetime)}</Header>
|
||||
{domainHeader}
|
||||
{urlHeader}
|
||||
|
||||
|
|
@ -159,6 +160,10 @@ function ArchivePage() {
|
|||
{deleteButton}
|
||||
</Segment>
|
||||
|
||||
<Segment>
|
||||
<TagsDisplay fileGroup={archiveFile} onClick={fetchArchive}/>
|
||||
</Segment>
|
||||
|
||||
<Segment>
|
||||
<HelpHeader
|
||||
headerContent='History'
|
||||
|
|
@ -171,17 +176,18 @@ function ArchivePage() {
|
|||
|
||||
export function ArchiveCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const {archive} = file;
|
||||
const {data} = file;
|
||||
|
||||
const imageSrc = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
|
||||
const singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null;
|
||||
const imageSrc = data.screenshot_path ? `/media/${encodeURIComponent(data.screenshot_path)}` : null;
|
||||
const singlefileUrl = data.singlefile_path ? `/media/${encodeURIComponent(data.singlefile_path)}` : null;
|
||||
|
||||
let screenshot = <CardIcon><FileIcon file={file}/></CardIcon>;
|
||||
const imageLabel = file.tags && file.tags.length ? taggedImageLabel : null;
|
||||
if (imageSrc) {
|
||||
screenshot = <Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>;
|
||||
screenshot = <Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}} label={imageLabel}/>;
|
||||
}
|
||||
|
||||
const domain = archive.domain ? archive.domain.domain : null;
|
||||
const domain = data ? data.domain : null;
|
||||
const domainUrl = `/archive?domain=${domain}`;
|
||||
|
||||
return <Card color={mimetypeColor(file.mimetype)}>
|
||||
|
|
@ -194,7 +200,7 @@ export function ArchiveCard({file}) {
|
|||
<CardHeader>
|
||||
<Container textAlign='left'>
|
||||
<ExternalCardLink to={singlefileUrl}>
|
||||
{cardTitleWrapper(archive.title || archive.url)}
|
||||
{cardTitleWrapper(file.title || data.url)}
|
||||
</ExternalCardLink>
|
||||
</Container>
|
||||
</CardHeader>
|
||||
|
|
@ -204,15 +210,15 @@ export function ArchiveCard({file}) {
|
|||
</CardLink>}
|
||||
<CardMeta {...s}>
|
||||
<p>
|
||||
{isoDatetimeToString(archive.archive_datetime)}
|
||||
{isoDatetimeToString(data.archive_datetime)}
|
||||
</p>
|
||||
</CardMeta>
|
||||
<CardDescription>
|
||||
<Link to={`/archive/${archive.id}`}>
|
||||
<Link to={`/archive/${data.id}`}>
|
||||
<Button icon='file alternate' content='Details'
|
||||
labelPosition='left'/>
|
||||
</Link>
|
||||
<Button icon='external' href={archive.url} target='_blank' rel='noopener noreferrer'/>
|
||||
<Button icon='external' href={data.url} target='_blank' rel='noopener noreferrer'/>
|
||||
</CardDescription>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
|
@ -427,10 +433,10 @@ function Archives() {
|
|||
}
|
||||
|
||||
export function ArchiveRowCells({file}) {
|
||||
const {archive} = file;
|
||||
const {data} = file;
|
||||
|
||||
const archiveUrl = `/archive/${archive.id}`;
|
||||
const posterUrl = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
|
||||
const archiveUrl = `/archive/${data.id}`;
|
||||
const posterUrl = data.screenshot_path ? `/media/${encodeURIComponent(data.screenshot_path)}` : null;
|
||||
|
||||
let poster;
|
||||
if (posterUrl) {
|
||||
|
|
@ -448,7 +454,8 @@ export function ArchiveRowCells({file}) {
|
|||
</TableCell>
|
||||
<TableCell>
|
||||
<CardLink to={archiveUrl}>
|
||||
{textEllipsis(archive.title || archive.stem)}
|
||||
<FileRowTagIcon file={file}/>
|
||||
{textEllipsis(file.title || file.stem)}
|
||||
</CardLink>
|
||||
</TableCell>
|
||||
</React.Fragment>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import React, {useContext, useEffect, useState} from "react";
|
||||
import {Card, Container, IconGroup, Input, Modal, Pagination} from 'semantic-ui-react';
|
||||
import {Card, Container, IconGroup, Image, Input, Modal, Pagination} from 'semantic-ui-react';
|
||||
import {Link, NavLink, useNavigate} from "react-router-dom";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {useDirectories, useDownloaders, useHotspot, useSettings, useThrottle} from "../hooks/customHooks";
|
||||
|
|
@ -117,7 +117,7 @@ export function Duration({video}) {
|
|||
|
||||
export function isoDatetimeToString(dt) {
|
||||
// Convert a datetime to a human-readable date format.
|
||||
let d = <></>;
|
||||
let d = <React.Fragment/>;
|
||||
if (dt) {
|
||||
d = new Date(dt);
|
||||
d = `${d.getFullYear()}-${d.getMonth() + 1}-${d.getDate()}`;
|
||||
|
|
@ -444,8 +444,8 @@ export function CardGroupCentered(props) {
|
|||
export function CardPosterLink({to, poster_url, imageLabel, external = false}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const style = {display: 'flex', justifyContent: 'center', ...s['style']};
|
||||
const image = <img src={poster_url} alt={imageLabel}
|
||||
style={{maxHeight: '163px', width: 'auto'}}
|
||||
const image = <Image src={poster_url} label={imageLabel}
|
||||
style={{maxHeight: '163px', width: 'auto'}}
|
||||
/>;
|
||||
if (external === true && to) {
|
||||
return <a href={to} target='_blank' rel='noreferrer' style={style}>
|
||||
|
|
@ -461,12 +461,12 @@ export function CardPosterLink({to, poster_url, imageLabel, external = false}) {
|
|||
}
|
||||
}
|
||||
|
||||
export function HelpPopup({icon, size, content, position}) {
|
||||
export function HelpPopup({icon, size, content, position, iconSize = 'small'}) {
|
||||
return <Popup
|
||||
content={content}
|
||||
size={size || null}
|
||||
position={position || 'left center'}
|
||||
trigger={<Icon circular link name={icon || 'question'} size='small'
|
||||
trigger={<Icon circular link name={icon || 'question'} size={iconSize}
|
||||
style={{marginLeft: '0.25em', marginRight: '0.25em'}}
|
||||
/>}
|
||||
/>
|
||||
|
|
@ -616,8 +616,9 @@ export function isZipMimetype(mimetype) {
|
|||
|
||||
export function FileIcon({file, disabled = true, size = 'huge', ...props}) {
|
||||
// Default to a grey file icon.
|
||||
const {mimetype, path} = file;
|
||||
const lowerPath = path.toLowerCase();
|
||||
const {mimetype, path, primary_path} = file;
|
||||
// `file` may be a file_group or a file.
|
||||
const lowerPath = primary_path ? primary_path.toLocaleString() : path.toLowerCase();
|
||||
props['name'] = 'file';
|
||||
props['color'] = mimetypeColor(mimetype);
|
||||
if (mimetype) {
|
||||
|
|
@ -851,3 +852,25 @@ export const cardTitleWrapper = (title, maxLength = 100, breakWord = true) => {
|
|||
const style = breakWord ? {overflowWrap: 'break-word'} : null;
|
||||
return <span style={style}>{textEllipsis(title, maxLength)}</span>
|
||||
}
|
||||
|
||||
function luma(color) {
|
||||
let rgb = (typeof color === 'string') ? hexToRGBArray(color) : color;
|
||||
return (0.2126 * rgb[0]) + (0.7152 * rgb[1]) + (0.0722 * rgb[2]); // SMPTE C, Rec. 709 weightings
|
||||
}
|
||||
|
||||
function hexToRGBArray(color) {
|
||||
if (color.length === 7) {
|
||||
color = color.slice(1);
|
||||
}
|
||||
if (color.length !== 6) {
|
||||
console.error('Invalid hex color: ' + color);
|
||||
return;
|
||||
}
|
||||
let rgb = [];
|
||||
for (let i = 0; i <= 2; i++) rgb[i] = parseInt(color.substr(i * 2, 2), 16);
|
||||
return rgb;
|
||||
}
|
||||
|
||||
export function contrastingColor(color) {
|
||||
return (luma(color) >= 120) ? '#000000' : '#dddddd';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
Container,
|
||||
Dropdown,
|
||||
Image,
|
||||
Label,
|
||||
PlaceholderLine,
|
||||
TableCell,
|
||||
TableHeaderCell,
|
||||
|
|
@ -40,6 +41,7 @@ import {FileBrowser} from "./FileBrowser";
|
|||
import {refreshDirectoryFiles, refreshFiles} from "../api";
|
||||
import {useSubscribeEventName} from "../Events";
|
||||
import {FilePreviewContext} from "./FilePreview";
|
||||
import {TagsContext} from "../Tags";
|
||||
|
||||
|
||||
export function FilesPage() {
|
||||
|
|
@ -50,20 +52,18 @@ export function FilesPage() {
|
|||
|
||||
function EbookCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
let {ebook, suffix} = file;
|
||||
let {data} = file;
|
||||
|
||||
const downloadUrl = `/download/${encodeURIComponent(file.path)}`;
|
||||
const downloadUrl = `/download/${encodeURIComponent(file.primary_path)}`;
|
||||
const isEpub = file['mimetype'].startsWith('application/epub');
|
||||
const viewerUrl = isEpub ? `/epub.html?url=${downloadUrl}` : null;
|
||||
|
||||
let cover = <FileIcon file={file}/>;
|
||||
if (ebook && ebook.cover_path) {
|
||||
const coverSrc = `/media/${encodeURIComponent(ebook.cover_path)}`;
|
||||
if (data && data.cover_path) {
|
||||
const coverSrc = `/media/${encodeURIComponent(data.cover_path)}`;
|
||||
cover = <CardPosterLink poster_url={coverSrc}/>;
|
||||
}
|
||||
|
||||
suffix = suffix ? _.trimStart(suffix, '.').toUpperCase() : null;
|
||||
|
||||
const color = mimetypeColor(file.mimetype);
|
||||
return <Card color={color}>
|
||||
<PreviewLink file={file}>
|
||||
|
|
@ -73,12 +73,13 @@ function EbookCard({file}) {
|
|||
<CardHeader>
|
||||
<Container textAlign='left'>
|
||||
<ExternalCardLink to={viewerUrl || downloadUrl}>
|
||||
{ebook ? ebook.title : file.title}
|
||||
{data ? data.title : file.title}
|
||||
</ExternalCardLink>
|
||||
</Container>
|
||||
</CardHeader>
|
||||
<CardMeta>
|
||||
<pre {...s}>{suffix}</pre>
|
||||
{data.creator ? <b {...s}>{data.creator}</b> : null}
|
||||
{file.size && <p {...s}>{humanFileSize(file.size)}</p>}
|
||||
</CardMeta>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
|
@ -88,7 +89,7 @@ function EbookCard({file}) {
|
|||
function ImageCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const {setPreviewFile} = React.useContext(FilePreviewContext);
|
||||
const url = `/media/${encodeURIComponent(file.path)}`;
|
||||
const url = `/media/${encodeURIComponent(file.primary_path)}`;
|
||||
|
||||
let poster = <FileIcon file={file}/>;
|
||||
if (file.size && file.size < 50000000) {
|
||||
|
|
@ -107,7 +108,7 @@ function ImageCard({file}) {
|
|||
<CardContent {...s}>
|
||||
<CardHeader>
|
||||
<ExternalCardLink to={url} className='no-link-underscore card-link'>
|
||||
<p>{textEllipsis(file.title || file.stem || file.path)}</p>
|
||||
<p>{textEllipsis(file.title || file.stem || file.primary_path)}</p>
|
||||
</ExternalCardLink>
|
||||
</CardHeader>
|
||||
<CardMeta {...s}>
|
||||
|
|
@ -127,17 +128,19 @@ function FileCard({file}) {
|
|||
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
|
||||
);
|
||||
|
||||
if (file.model === 'video' && file['video']) {
|
||||
return <VideoCard key={file['path']} file={file}/>;
|
||||
} else if (file.model === 'archive' && file['archive']) {
|
||||
return <ArchiveCard key={file['path']} file={file}/>;
|
||||
if (file.model === 'video') {
|
||||
return <VideoCard key={file['primary_path']} file={file}/>;
|
||||
} else if (file.model === 'archive') {
|
||||
return <ArchiveCard key={file['primary_path']} file={file}/>;
|
||||
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
|
||||
return <ImageCard key={file['path']} file={file}/>;
|
||||
return <ImageCard key={file['primary_path']} file={file}/>;
|
||||
} else if (isEbookType) {
|
||||
return <EbookCard key={file['path']} file={file}/>;
|
||||
return <EbookCard key={file['primary_path']} file={file}/>;
|
||||
}
|
||||
|
||||
const downloadUrl = `/download/${encodeURIComponent(file.path)}`;
|
||||
const {data} = file;
|
||||
const {author} = data;
|
||||
const downloadUrl = `/download/${encodeURIComponent(file.primary_path)}`;
|
||||
const color = mimetypeColor(file.mimetype);
|
||||
const size = file.size !== null && file.size !== undefined ? humanFileSize(file.size) : null;
|
||||
|
||||
|
|
@ -150,10 +153,11 @@ function FileCard({file}) {
|
|||
<CardContent {...s}>
|
||||
<CardHeader>
|
||||
<ExternalCardLink to={downloadUrl}>
|
||||
{cardTitleWrapper(file.title || file.stem || file.path)}
|
||||
{cardTitleWrapper(file.title || file.stem || file.primary_path)}
|
||||
</ExternalCardLink>
|
||||
</CardHeader>
|
||||
<p>{isoDatetimeToString(file.modified / 1000)}</p>
|
||||
{author && <b {...s}>{author}</b>}
|
||||
<p>{isoDatetimeToString(file.modified)}</p>
|
||||
<p>{size}</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
|
@ -162,7 +166,7 @@ function FileCard({file}) {
|
|||
export function FileCards({files}) {
|
||||
if (!_.isEmpty(files)) {
|
||||
return <CardGroupCentered>
|
||||
{files.map(i => <FileCard key={i['path']} file={i}/>)}
|
||||
{files.map(i => <FileCard key={i['primary_path']} file={i}/>)}
|
||||
</CardGroupCentered>
|
||||
} else if (files && files.length === 0) {
|
||||
return <Segment>No results!</Segment>
|
||||
|
|
@ -172,7 +176,7 @@ export function FileCards({files}) {
|
|||
}
|
||||
|
||||
function ImageRowCells({file}) {
|
||||
const url = `/media/${encodeURIComponent(file.path)}`;
|
||||
const url = `/media/${encodeURIComponent(file.primary_path)}`;
|
||||
|
||||
let poster = <FileIcon file={file} size='large'/>;
|
||||
if (file.size && file.size < 50000000) {
|
||||
|
|
@ -189,18 +193,18 @@ function ImageRowCells({file}) {
|
|||
</TableCell>
|
||||
<TableCell>
|
||||
<PreviewLink file={file}>
|
||||
<p>{textEllipsis(file.title || file.stem || file.path)}</p>
|
||||
<p>{textEllipsis(file.title || file.stem || file.primary_path)}</p>
|
||||
</PreviewLink>
|
||||
</TableCell>
|
||||
</React.Fragment>
|
||||
}
|
||||
|
||||
export function EbookRowCells({file}) {
|
||||
const {ebook} = file;
|
||||
const {data} = file;
|
||||
|
||||
let cover = <CardIcon><FileIcon file={file}/></CardIcon>;
|
||||
if (ebook && ebook.cover_path) {
|
||||
const coverSrc = `/media/${encodeURIComponent(ebook.cover_path)}`;
|
||||
if (data && data.cover_path) {
|
||||
const coverSrc = `/media/${encodeURIComponent(data.cover_path)}`;
|
||||
cover = <Image wrapped src={coverSrc} width='50px'/>;
|
||||
}
|
||||
|
||||
|
|
@ -211,7 +215,7 @@ export function EbookRowCells({file}) {
|
|||
</TableCell>
|
||||
<TableCell>
|
||||
<PreviewLink file={file}>
|
||||
{textEllipsis(ebook.title || file.title)}
|
||||
{textEllipsis(file.title || file.stem)}
|
||||
</PreviewLink>
|
||||
</TableCell>
|
||||
</React.Fragment>
|
||||
|
|
@ -224,12 +228,12 @@ function FileRow({file}) {
|
|||
|
||||
if (file.model === 'video' && 'video' in file) {
|
||||
return <VideoRowCells file={file}/>;
|
||||
} else if (file.model === 'archive' && 'archive' in file) {
|
||||
} else if (file.model === 'archive') {
|
||||
return <ArchiveRowCells file={file}/>;
|
||||
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
|
||||
return <ImageRowCells file={file}/>;
|
||||
} else if (isEbookType) {
|
||||
return <EbookRowCells key={file['path']} file={file}/>;
|
||||
return <EbookRowCells key={file['primary_path']} file={file}/>;
|
||||
}
|
||||
|
||||
// Fragment for SelectableRow
|
||||
|
|
@ -266,6 +270,14 @@ export function FileTable({files, selectOn, onSelect, footer, selectedKeys}) {
|
|||
}
|
||||
}
|
||||
|
||||
export function FileRowTagIcon({file}) {
|
||||
if (file.tags && file.tags.length) {
|
||||
return <Label circular color='green' style={{padding: '0.5em', marginRight: '0.5em'}}>
|
||||
<Icon name='tag' style={{margin: 0}}/>
|
||||
</Label>;
|
||||
}
|
||||
}
|
||||
|
||||
export function FilesView({
|
||||
files,
|
||||
limit,
|
||||
|
|
@ -347,19 +359,20 @@ export function FilesView({
|
|||
|
||||
const filtersDropdown = <Dropdown selection clearable search
|
||||
multiple={multipleFilters || undefined}
|
||||
options={filterOptions}
|
||||
placeholder={filterPlaceholder || 'Filters'}
|
||||
options={filterOptions || []}
|
||||
placeholder={filterPlaceholder || 'Files Types'}
|
||||
onChange={(e, {value}) => setFilters(value)}
|
||||
style={{marginLeft: '0.3em'}}
|
||||
style={{marginLeft: '0.3em', marginTop: '0.3em'}}
|
||||
value={activeFilters}
|
||||
/>;
|
||||
|
||||
return <>
|
||||
<Grid columns={menuColumnsCount || 1} stackable>
|
||||
<Grid.Column mobile={8} computer={6}>
|
||||
<Grid.Column mobile={16} computer={12}>
|
||||
{selectButton}
|
||||
{viewButton}
|
||||
{limitDropdown}
|
||||
<TagsFilesGroupDropdown/>
|
||||
{filterOptions && filtersDropdown}
|
||||
</Grid.Column>
|
||||
{menuColumns}
|
||||
|
|
@ -369,6 +382,38 @@ export function FilesView({
|
|||
</>
|
||||
}
|
||||
|
||||
export function TagsFilesGroupDropdown({onChange}) {
|
||||
// Creates a dropdown that the User can use to manipulate the tag query.
|
||||
const {searchParams, updateQuery} = useQuery();
|
||||
const activeTag = searchParams.get('tag');
|
||||
const {tags} = React.useContext(TagsContext);
|
||||
|
||||
const localOnChange = (name) => {
|
||||
updateQuery({'tag': name});
|
||||
if (onChange) {
|
||||
onChange(name);
|
||||
}
|
||||
}
|
||||
|
||||
let tagOptions = [];
|
||||
if (tags && tags.length) {
|
||||
for (let i = 0; i < tags.length; i++) {
|
||||
const {name} = tags[i];
|
||||
tagOptions = [...tagOptions, {key: name, text: name, value: name}];
|
||||
}
|
||||
|
||||
return <Dropdown selection clearable search
|
||||
options={tagOptions || []}
|
||||
placeholder={'Tags'}
|
||||
onChange={(e, {value}) => localOnChange(value)}
|
||||
style={{marginLeft: '0.3em', marginTop: '0.3em'}}
|
||||
value={activeTag}
|
||||
/>;
|
||||
} else {
|
||||
return <React.Fragment/>
|
||||
}
|
||||
}
|
||||
|
||||
export function FilesSearchView({
|
||||
showView = true,
|
||||
showLimit = false,
|
||||
|
|
@ -391,7 +436,6 @@ export function FilesSearchView({
|
|||
|
||||
const {searchFiles, limit, setLimit, totalPages, activePage, setPage} =
|
||||
useSearchFiles(24, emptySearch, model);
|
||||
|
||||
const {searchParams, updateQuery} = useQuery();
|
||||
const setView = (value) => updateQuery({view: value});
|
||||
const view = searchParams.get('view');
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {useChannel} from "../hooks/customHooks";
|
|||
import {ThemeContext} from "../contexts/contexts";
|
||||
import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme";
|
||||
import {VideoCard} from "./Videos";
|
||||
import {TagsDisplay} from "../Tags";
|
||||
|
||||
const MEDIA_PATH = '/media';
|
||||
|
||||
|
|
@ -26,6 +27,7 @@ const MEDIA_PATH = '/media';
|
|||
function videoFileLink(path, directory = false) {
|
||||
if (path) {
|
||||
const href = directory ?
|
||||
// Add / to end of directory.
|
||||
`${MEDIA_PATH}/${encodeURIComponent(path)}/` :
|
||||
`${MEDIA_PATH}/${encodeURIComponent(path)}`;
|
||||
return <a href={href} target='_blank' rel='noopener noreferrer'>
|
||||
|
|
@ -36,7 +38,7 @@ function videoFileLink(path, directory = false) {
|
|||
}
|
||||
}
|
||||
|
||||
function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
|
||||
function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
||||
const {theme} = useContext(ThemeContext);
|
||||
|
||||
const navigate = useNavigate();
|
||||
|
|
@ -84,30 +86,12 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
|
|||
let vttCaptions = video.caption_path && video.caption_path.endsWith('.vtt');
|
||||
|
||||
let description = 'No description available.';
|
||||
let viewCount = video.view_count;
|
||||
if (video.info_json && video.info_json['description']) {
|
||||
if (video && video['description']) {
|
||||
// Only replace empty description if there is one available.
|
||||
description = video.info_json['description'];
|
||||
description = video['description'];
|
||||
description = chaptersInDescription(description, setVideoTime)
|
||||
viewCount = viewCount || video.info_json['view_count'];
|
||||
}
|
||||
|
||||
const favorite = video && video.favorite;
|
||||
let handleFavorite = async (e) => {
|
||||
e.preventDefault();
|
||||
if (video) {
|
||||
setFavorite(!favorite);
|
||||
}
|
||||
}
|
||||
const favorite_button = (<Button
|
||||
color='green'
|
||||
style={{'margin': '0.5em'}}
|
||||
onClick={handleFavorite}
|
||||
>
|
||||
<Icon name='heart'/>
|
||||
{favorite ? 'Unfavorite' : 'Favorite'}
|
||||
</Button>);
|
||||
|
||||
let descriptionPane = {
|
||||
menuItem: 'Description', render: () => <TabPane>
|
||||
<pre className="wrap-text">
|
||||
|
|
@ -119,13 +103,13 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
|
|||
let aboutPane = {
|
||||
menuItem: 'About', render: () => <TabPane>
|
||||
<h3>Size</h3>
|
||||
<p>{video.size ? humanFileSize(video.size) : 'Unknown'}</p>
|
||||
<p>{videoFile.size ? humanFileSize(videoFile.size) : 'Unknown'}</p>
|
||||
|
||||
<h3>Source URL</h3>
|
||||
<p>{video.url ? <a href={video.url}>{video.url}</a> : 'N/A'}</p>
|
||||
|
||||
<h3>View Count</h3>
|
||||
<p>{viewCount ? humanNumber(viewCount) : 'N/A'}</p>
|
||||
<p>{video.view_count ? humanNumber(video.view_count) : 'N/A'}</p>
|
||||
|
||||
<h3>Censored</h3>
|
||||
<p>{video.censored ? 'Yes' : 'No'}</p>
|
||||
|
|
@ -138,25 +122,30 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
|
|||
</TabPane>
|
||||
};
|
||||
|
||||
const {poster_file, caption_files, info_json_file} = video;
|
||||
let filesPane = {
|
||||
menuItem: 'Files', render: () => <TabPane>
|
||||
<h3>Video File</h3>
|
||||
{videoFileLink(video['video_path'])}
|
||||
|
||||
<h4>Info JSON File</h4>
|
||||
<PreviewPath
|
||||
path={video['info_json_path']}
|
||||
mimetype='application/json'>{video['info_json_path']}</PreviewPath>
|
||||
{info_json_file &&
|
||||
<PreviewPath
|
||||
path={info_json_file['path']}
|
||||
mimetype={info_json_file['mimetype']}>{info_json_file['path']}</PreviewPath>
|
||||
}
|
||||
|
||||
<h4>Caption File</h4>
|
||||
<PreviewPath
|
||||
path={video['caption_path']}
|
||||
mimetype='text/'>{video['caption_path']}</PreviewPath>
|
||||
<h4>Caption Files</h4>
|
||||
{caption_files &&
|
||||
caption_files.map(i => <PreviewPath key={i['path']} {...i}>{i['path']}</PreviewPath>)
|
||||
}
|
||||
|
||||
<h4>Poster File</h4>
|
||||
<PreviewPath
|
||||
path={video['poster_path']}
|
||||
mimetype='image/'>{video['poster_path']}</PreviewPath>
|
||||
{poster_file &&
|
||||
<PreviewPath
|
||||
path={poster_file['path']}
|
||||
mimetype={poster_file['mimetype']}>{poster_file['path']}</PreviewPath>
|
||||
}
|
||||
|
||||
<h4>Directory</h4>
|
||||
{videoFileLink(videoFile['directory'], true)}
|
||||
|
|
@ -188,7 +177,7 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
|
|||
<Container style={{marginTop: '1em'}}>
|
||||
<Segment>
|
||||
|
||||
<Header as='h2'>{video.title || video.video_path}</Header>
|
||||
<Header as='h2'>{videoFile.title || videoFile.stem || video.video_path}</Header>
|
||||
{video.upload_date && <h3>{isoDatetimeToString(video.upload_date)}</h3>}
|
||||
<h3>
|
||||
{channel && <Link to={`/videos/channel/${channel.id}/video`}>
|
||||
|
|
@ -197,7 +186,6 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
|
|||
</h3>
|
||||
|
||||
<p>
|
||||
{favorite_button}
|
||||
<a href={downloadUrl} target='_blank' rel='noopener noreferrer'>
|
||||
<Button style={{margin: '0.5em'}}>
|
||||
<Icon name='download'/>
|
||||
|
|
@ -217,6 +205,11 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
|
|||
onConfirm={() => handleDeleteVideo(video.id)}
|
||||
/>
|
||||
</p>
|
||||
<br/>
|
||||
</Segment>
|
||||
|
||||
<Segment>
|
||||
<TagsDisplay fileGroup={videoFile} onClick={fetchVideo}/>
|
||||
</Segment>
|
||||
|
||||
<Tab menu={tabMenu} panes={tabPanes}/>
|
||||
|
|
|
|||
|
|
@ -34,26 +34,27 @@ import {
|
|||
} from "semantic-ui-react";
|
||||
import {Channels, EditChannel, NewChannel} from "./Channels";
|
||||
import {useChannel, useQuery, useSearchVideos, useVideo, useVideoStatistics} from "../hooks/customHooks";
|
||||
import {FilesView} from "./Files";
|
||||
import {FileRowTagIcon, FilesView} from "./Files";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
|
||||
import {Button, Card, CardIcon, Header, Loader, Placeholder, Segment, Statistic, StatisticGroup} from "./Theme";
|
||||
import {deleteVideos} from "../api";
|
||||
import {ThemeContext} from "../contexts/contexts";
|
||||
import _ from "lodash";
|
||||
import {taggedImageLabel} from "../Tags";
|
||||
|
||||
export function VideoWrapper() {
|
||||
const {videoId} = useParams();
|
||||
const {videoFile, prevFile, nextFile, setFavorite} = useVideo(videoId);
|
||||
const {videoFile, prevFile, nextFile, fetchVideo} = useVideo(videoId);
|
||||
|
||||
// Scroll to the top when videoId changes.
|
||||
useEffect(scrollToTop, [videoId]);
|
||||
|
||||
return <VideoPage videoFile={videoFile} prevFile={prevFile} nextFile={nextFile}
|
||||
setFavorite={setFavorite} autoplay={true}/>
|
||||
return <VideoPage videoFile={videoFile} prevFile={prevFile} nextFile={nextFile} fetchVideo={fetchVideo}
|
||||
autoplay={true}/>
|
||||
}
|
||||
|
||||
function VideosPage({filter}) {
|
||||
function VideosPage() {
|
||||
|
||||
const {channelId} = useParams();
|
||||
const {searchParams, updateQuery} = useQuery();
|
||||
|
|
@ -69,31 +70,19 @@ function VideosPage({filter}) {
|
|||
searchOrder = defaultSearchOrder;
|
||||
}
|
||||
|
||||
let filtersEnabled = searchParams.getAll('filter');
|
||||
filtersEnabled = filter ? [...filtersEnabled, filter] : filtersEnabled;
|
||||
|
||||
const {searchStr, setSearchStr, videos, activePage, setPage, limit, setLimit, totalPages, setOrderBy, fetchVideos} =
|
||||
useSearchVideos(null, channelId, searchOrder, filtersEnabled);
|
||||
useSearchVideos(null, channelId, searchOrder);
|
||||
const setView = (value) => updateQuery({view: value});
|
||||
const view = searchParams.get('view');
|
||||
|
||||
const {channel} = useChannel(channelId);
|
||||
|
||||
let title = 'Videos';
|
||||
if (filter && filter.indexOf('favorite') >= 0) {
|
||||
title = 'Favorite Videos';
|
||||
}
|
||||
if (channel && channel.name) {
|
||||
title = `${channel.name} Videos`;
|
||||
}
|
||||
useTitle(title);
|
||||
|
||||
const filterOptions = [
|
||||
{text: 'Favorites', key: 'favorite', value: 'favorite'},
|
||||
{text: 'Censored', key: 'censored', value: 'censored'},
|
||||
];
|
||||
const setFilters = (value) => updateQuery({filter: value});
|
||||
|
||||
let videoOrders = [
|
||||
{key: '-upload_date', value: '-upload_date', text: 'Newest'},
|
||||
{key: 'upload_date', value: 'upload_date', text: 'Oldest'},
|
||||
|
|
@ -219,10 +208,6 @@ function VideosPage({filter}) {
|
|||
setView={setView}
|
||||
setLimit={setLimit}
|
||||
setPage={setPage}
|
||||
filterOptions={filterOptions}
|
||||
activeFilters={filtersEnabled}
|
||||
setFilters={setFilters}
|
||||
multipleFilters={true}
|
||||
menuColumnsCount={3}
|
||||
menuColumns={menuColumns}
|
||||
>
|
||||
|
|
@ -242,7 +227,6 @@ function Statistics() {
|
|||
|
||||
const videoNames = [
|
||||
{key: 'videos', label: 'Videos'},
|
||||
{key: 'favorites', label: 'Favorite Videos'},
|
||||
{key: 'sum_size', label: 'Total Size'},
|
||||
{key: 'max_size', label: 'Largest Video'},
|
||||
{key: 'week', label: 'Downloads Past Week'},
|
||||
|
|
@ -283,7 +267,6 @@ function Statistics() {
|
|||
export function VideosRoute(props) {
|
||||
const links = [
|
||||
{text: 'Videos', to: '/videos', key: 'videos', end: true},
|
||||
{text: 'Favorites', to: '/videos/favorites', key: 'favorites'},
|
||||
{text: 'Channels', to: '/videos/channel', key: 'channel'},
|
||||
{text: 'Statistics', to: '/videos/statistics', key: 'statistics'},
|
||||
];
|
||||
|
|
@ -292,7 +275,6 @@ export function VideosRoute(props) {
|
|||
<TabLinks links={links}/>
|
||||
<Routes>
|
||||
<Route path='/' exact element={<VideosPage/>}/>
|
||||
<Route path='favorites' exact element={<VideosPage filter='favorite' header='Favorite Videos'/>}/>
|
||||
<Route path='channel' exact element={<Channels/>}/>
|
||||
<Route path='statistics' exact element={<Statistics/>}/>
|
||||
<Route path='channel/new' exact element={<NewChannel/>}/>
|
||||
|
|
@ -303,7 +285,7 @@ export function VideosRoute(props) {
|
|||
}
|
||||
|
||||
export function VideoCard({file}) {
|
||||
const {video} = file;
|
||||
const {video, tags} = file;
|
||||
const {s} = useContext(ThemeContext);
|
||||
|
||||
let video_url = `/videos/video/${video.id}`;
|
||||
|
|
@ -317,10 +299,7 @@ export function VideoCard({file}) {
|
|||
}
|
||||
const poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;
|
||||
|
||||
let imageLabel = null;
|
||||
if (video.favorite) {
|
||||
imageLabel = {corner: 'left', icon: 'heart', color: 'green'};
|
||||
}
|
||||
let imageLabel = !_.isEmpty(tags) ? taggedImageLabel : null;
|
||||
let poster = <Link to={video_url}><CardIcon><FileIcon file={file}/></CardIcon></Link>;
|
||||
if (poster_url) {
|
||||
poster = <CardPosterLink to={video_url} poster_url={poster_url} imageLabel={imageLabel}/>;
|
||||
|
|
@ -334,7 +313,7 @@ export function VideoCard({file}) {
|
|||
<CardHeader>
|
||||
<Container textAlign='left'>
|
||||
<Link to={video_url} className="no-link-underscore card-link">
|
||||
<p {...s}>{cardTitleWrapper(video.title || video.stem || video.video_path)}</p>
|
||||
<p {...s}>{cardTitleWrapper(file.title || file.name || video.stem || video.video_path)}</p>
|
||||
</Link>
|
||||
</Container>
|
||||
</CardHeader>
|
||||
|
|
@ -357,12 +336,10 @@ export function VideoRowCells({file}) {
|
|||
const poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;
|
||||
|
||||
let poster;
|
||||
let imageLabel = video.favorite ? {corner: 'left', icon: 'heart', color: 'green'} : null;
|
||||
if (poster_url) {
|
||||
poster = <CardLink to={video_url}>
|
||||
<Image wrapped
|
||||
src={poster_url}
|
||||
label={imageLabel}
|
||||
width='50px'
|
||||
/>
|
||||
</CardLink>
|
||||
|
|
@ -379,6 +356,7 @@ export function VideoRowCells({file}) {
|
|||
</TableCell>
|
||||
<TableCell>
|
||||
<CardLink to={video_url}>
|
||||
<FileRowTagIcon file={file}/>
|
||||
{textEllipsis(video.title || video.stem || video.video_path)}
|
||||
</CardLink>
|
||||
</TableCell>
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class WROLMode extends React.Component {
|
|||
want to prevent any potential loss of data.
|
||||
</Header>
|
||||
<p {...t}>
|
||||
Note: User settings and favorites can still be modified.
|
||||
Note: User settings and tags can still be modified.
|
||||
</p>
|
||||
<Toggle
|
||||
checked={this.state.WROLMode}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import {useContext, useEffect, useRef, useState} from "react";
|
||||
import {
|
||||
favoriteVideo,
|
||||
fetchDomains,
|
||||
filesSearch,
|
||||
getArchive,
|
||||
|
|
@ -81,17 +80,17 @@ export const useDomains = () => {
|
|||
}
|
||||
|
||||
export const useArchive = (archiveId) => {
|
||||
const [archiveFile, setArchiveFile] = useState(null);
|
||||
const [archiveFileGroup, setArchiveFileGroup] = useState(null);
|
||||
const [history, setHistory] = useState(null);
|
||||
|
||||
const fetchArchive = async () => {
|
||||
try {
|
||||
const [file, history] = await getArchive(archiveId);
|
||||
setArchiveFile(file);
|
||||
const [file_group, history] = await getArchive(archiveId);
|
||||
setArchiveFileGroup(file_group);
|
||||
setHistory(history);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
setArchiveFile(undefined);
|
||||
setArchiveFileGroup(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -99,7 +98,7 @@ export const useArchive = (archiveId) => {
|
|||
fetchArchive();
|
||||
}, [archiveId]);
|
||||
|
||||
return {archiveFile, history};
|
||||
return {archiveFile: archiveFileGroup, history, fetchArchive};
|
||||
}
|
||||
|
||||
export const useQuery = () => {
|
||||
|
|
@ -152,6 +151,7 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
|
|||
const {searchParams, updateQuery} = useQuery();
|
||||
const searchStr = searchParams.get('q') || '';
|
||||
const order = searchParams.get('order') || order_by;
|
||||
const activeTags = searchParams.get('tag');
|
||||
|
||||
const [archives, setArchives] = useState(null);
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
|
|
@ -160,7 +160,7 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
|
|||
setArchives(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order);
|
||||
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags);
|
||||
setArchives(archives_);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
|
|
@ -177,7 +177,7 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
|
|||
|
||||
useEffect(() => {
|
||||
localSearchArchives();
|
||||
}, [searchStr, limit, domain, order, activePage]);
|
||||
}, [searchStr, limit, domain, order, activePage, activeTags]);
|
||||
|
||||
const setSearchStr = (value) => {
|
||||
updateQuery({q: value, o: 0, order: undefined});
|
||||
|
|
@ -204,11 +204,12 @@ export const useSearchArchives = (defaultLimit, domain, order_by) => {
|
|||
}
|
||||
}
|
||||
|
||||
export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
|
||||
export const useSearchVideos = (defaultLimit, channelId, order_by) => {
|
||||
const {searchParams, updateQuery} = useQuery();
|
||||
const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit)
|
||||
const searchStr = searchParams.get('q') || '';
|
||||
const order = searchParams.get('order') || order_by;
|
||||
const activeTags = searchParams.get('tag');
|
||||
|
||||
const [videos, setVideos] = useState();
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
|
|
@ -217,7 +218,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
|
|||
setVideos(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, filters);
|
||||
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags);
|
||||
setVideos(videos_);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
|
|
@ -234,7 +235,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
|
|||
|
||||
useEffect(() => {
|
||||
localSearchVideos();
|
||||
}, [searchStr, limit, channelId, offset, order_by, filters.join('')]);
|
||||
}, [searchStr, limit, channelId, offset, order_by, activeTags]);
|
||||
|
||||
const setSearchStr = (value) => {
|
||||
updateQuery({q: value, o: 0, order: undefined});
|
||||
|
|
@ -258,6 +259,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by, filters) => {
|
|||
setSearchStr,
|
||||
setOrderBy,
|
||||
fetchVideos: localSearchVideos,
|
||||
activeTags,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -292,24 +294,11 @@ export const useVideo = (videoId) => {
|
|||
}
|
||||
}
|
||||
|
||||
const setFavorite = async (value) => {
|
||||
if (!videoFile) {
|
||||
console.error('No video to favorite');
|
||||
return
|
||||
}
|
||||
try {
|
||||
await favoriteVideo(videoFile.video.id, value);
|
||||
fetchVideo();
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
fetchVideo();
|
||||
}, [videoId]);
|
||||
|
||||
return {videoFile, prevFile, nextFile, setFavorite};
|
||||
return {videoFile, prevFile, nextFile, fetchVideo};
|
||||
}
|
||||
|
||||
export const useChannel = (channel_id) => {
|
||||
|
|
@ -394,22 +383,23 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
|
|||
const searchStr = searchParams.get('q');
|
||||
const filter = searchParams.get('filter');
|
||||
const model_ = searchParams.get('model');
|
||||
const activeTags = searchParams.get('tag');
|
||||
|
||||
const [searchFiles, setSearchFiles] = useState(null);
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
const [activePage, setActivePage] = useState(calculatePage(offset, limit));
|
||||
|
||||
const localSearchFiles = async () => {
|
||||
if (!emptySearch && !searchStr) {
|
||||
if (!emptySearch && !searchStr && !activeTags) {
|
||||
return;
|
||||
}
|
||||
const mimetypes = filterToMimetypes(filter);
|
||||
setSearchFiles(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [files, total] = await filesSearch(
|
||||
offset, limit, searchStr, mimetypes, model || model_);
|
||||
setSearchFiles(files);
|
||||
let [file_groups, total] = await filesSearch(
|
||||
offset, limit, searchStr, mimetypes, model || model_, activeTags);
|
||||
setSearchFiles(file_groups);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
|
|
@ -424,7 +414,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
|
|||
|
||||
useEffect(() => {
|
||||
localSearchFiles();
|
||||
}, [searchStr, limit, offset, activePage, filter, model, model_]);
|
||||
}, [searchStr, limit, offset, activePage, filter, model, model_, activeTags]);
|
||||
|
||||
const setPage = (i) => {
|
||||
i = parseInt(i);
|
||||
|
|
@ -442,7 +432,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
|
|||
updateQuery({l: value, o: 0});
|
||||
}
|
||||
|
||||
return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit};
|
||||
return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit, activeTags};
|
||||
}
|
||||
|
||||
export const useBrowseFiles = () => {
|
||||
|
|
|
|||
|
|
@ -33,16 +33,16 @@ services:
|
|||
# healthcheck:
|
||||
# test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]
|
||||
|
||||
archive:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/archive/Dockerfile
|
||||
volumes:
|
||||
- './docker/archive/main.py:/app/main.py'
|
||||
ports:
|
||||
- '8083:8080'
|
||||
healthcheck:
|
||||
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
|
||||
# archive:
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: docker/archive/Dockerfile
|
||||
# volumes:
|
||||
# - './docker/archive/main.py:/app/main.py'
|
||||
# ports:
|
||||
# - '8083:8080'
|
||||
# healthcheck:
|
||||
# test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
|
||||
|
||||
app:
|
||||
depends_on:
|
||||
|
|
|
|||
32
main.py
32
main.py
|
|
@ -10,7 +10,7 @@ from sanic.signals import Event
|
|||
from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS
|
||||
from wrolpi import root_api, admin
|
||||
from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, \
|
||||
wrol_mode_enabled, cancel_refresh_tasks
|
||||
wrol_mode_enabled, cancel_refresh_tasks, set_log_level
|
||||
from wrolpi.downloader import download_manager, import_downloads_config
|
||||
from wrolpi.root_api import api_app
|
||||
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
|
||||
|
|
@ -115,8 +115,14 @@ def main():
|
|||
return 1
|
||||
|
||||
logger.warning(f'Starting with: {sys.argv}')
|
||||
set_log_level(args)
|
||||
logger.debug(get_version_string())
|
||||
if args.verbose == 1:
|
||||
set_log_level(logging.INFO)
|
||||
elif args.verbose and args.verbose == 2:
|
||||
set_log_level(logging.DEBUG)
|
||||
elif args.verbose and args.verbose >= 3:
|
||||
# Log everything. Add SQLAlchemy debug logging.
|
||||
set_log_level(logging.NOTSET)
|
||||
logger.info(get_version_string())
|
||||
|
||||
if DOCKERIZED:
|
||||
logger.info('Running in Docker')
|
||||
|
|
@ -151,26 +157,6 @@ def main():
|
|||
return root_api.main(args)
|
||||
|
||||
|
||||
def set_log_level(args):
|
||||
"""
|
||||
Set the level at the root logger so all children that have been created (or will be created) share the same level.
|
||||
"""
|
||||
root_logger = logging.getLogger()
|
||||
sa_logger = logging.getLogger('sqlalchemy.engine')
|
||||
if args.verbose == 1:
|
||||
root_logger.setLevel(logging.INFO)
|
||||
elif args.verbose and args.verbose == 2:
|
||||
root_logger.setLevel(logging.DEBUG)
|
||||
elif args.verbose and args.verbose >= 3:
|
||||
root_logger.setLevel(logging.DEBUG)
|
||||
sa_logger.setLevel(logging.DEBUG)
|
||||
|
||||
# Always warn about the log level so we know what will be logged
|
||||
effective_level = logger.getEffectiveLevel()
|
||||
level_name = logging.getLevelName(effective_level)
|
||||
logger.warning(f'Logging level: {level_name}')
|
||||
|
||||
|
||||
@api_app.before_server_start
|
||||
@limit_concurrent(1)
|
||||
async def startup(app: Sanic):
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
import asyncio
|
||||
import json
|
||||
import pathlib
|
||||
from abc import ABC
|
||||
from typing import Tuple, Dict, List
|
||||
from typing import Tuple, List
|
||||
|
||||
from sqlalchemy import not_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.indexers import register_indexer, Indexer
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.errors import UnrecoverableDownloadError, InvalidArchive
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
from . import lib
|
||||
from .api import bp # noqa
|
||||
from .lib import match_archive_files
|
||||
from .lib import is_singlefile_file, get_title_from_html
|
||||
from .models import Archive, Domain
|
||||
|
||||
PRETTY_NAME = 'Archive'
|
||||
|
|
@ -55,58 +55,129 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
archive_downloader = ArchiveDownloader(priority=100)
|
||||
|
||||
|
||||
def find_archive_file_in_group(group: List[File]):
|
||||
for file in group:
|
||||
if lib.is_singlefile_file(file.path):
|
||||
return file
|
||||
def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
||||
file_group_id = file_group.id
|
||||
if not file_group_id:
|
||||
session.flush([file_group])
|
||||
file_group_id = file_group.id
|
||||
|
||||
# All Archives have an HTML Singlefile.
|
||||
html_paths = file_group.my_paths('text/html')
|
||||
if not html_paths:
|
||||
logger.error('Query returned a group without an HTML file!')
|
||||
raise InvalidArchive('FileGroup does not contain any html files')
|
||||
|
||||
# All Archives have a Singlefile.
|
||||
for file in html_paths:
|
||||
if is_singlefile_file(file):
|
||||
singlefile_path = file
|
||||
break
|
||||
else:
|
||||
logger.debug(f'No Archive singlefile found in {file_group}')
|
||||
raise InvalidArchive('FileGroup does not contain a singlefile')
|
||||
|
||||
readability_json_path = None
|
||||
json_files = file_group.my_json_files()
|
||||
for json_file in json_files:
|
||||
path = json_file['path']
|
||||
if path.name.endswith('.readability.json'):
|
||||
readability_json_path = path
|
||||
break
|
||||
|
||||
readability_txt_path = None
|
||||
text_files = file_group.my_text_files()
|
||||
for text_file in text_files:
|
||||
path = text_file['path']
|
||||
if path.name.endswith('.readability.txt'):
|
||||
readability_txt_path = path
|
||||
break
|
||||
|
||||
try:
|
||||
file_group.model = 'archive'
|
||||
file_group.primary_path = singlefile_path
|
||||
|
||||
title = None
|
||||
if readability_json_path:
|
||||
title = get_title(readability_json_path)
|
||||
if not title:
|
||||
title = get_title_from_html(singlefile_path.read_text())
|
||||
|
||||
contents = None
|
||||
if readability_txt_path:
|
||||
contents = get_article(readability_txt_path)
|
||||
|
||||
archive = Archive(file_group_id=file_group_id, file_group=file_group)
|
||||
session.add(archive)
|
||||
archive.validate()
|
||||
session.flush([archive])
|
||||
|
||||
file_group.title = file_group.a_text = title
|
||||
file_group.d_text = contents
|
||||
file_group.data = {
|
||||
'id': archive.id,
|
||||
'archive_datetime': archive.archive_datetime,
|
||||
'domain': archive.domain.domain if archive.domain else None,
|
||||
'readability_json_path': archive.readability_json_path,
|
||||
'readability_path': archive.readability_path,
|
||||
'readability_txt_path': archive.readability_txt_path,
|
||||
'screenshot_path': archive.screenshot_path,
|
||||
'singlefile_path': archive.singlefile_path,
|
||||
'url': archive.url,
|
||||
}
|
||||
|
||||
return archive
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to model Archive {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
|
||||
@register_modeler
|
||||
def archive_modeler(groups: Dict[str, List[File]], session: Session):
|
||||
archive_files = {stem: file for stem, group in groups.items() if (file := find_archive_file_in_group(group))}
|
||||
if not archive_files:
|
||||
# No archives in these groups.
|
||||
return
|
||||
# Get all matching Archive records (if any) in one query.
|
||||
archive_paths = [i.path for i in archive_files.values()]
|
||||
archive_records = {i.singlefile_path: i for i in
|
||||
session.query(Archive).filter(Archive.singlefile_path.in_(archive_paths))}
|
||||
async def archive_modeler():
|
||||
invalid_archives = set()
|
||||
|
||||
for stem, archive_file in archive_files.items():
|
||||
group = groups[stem]
|
||||
while True:
|
||||
with get_db_session(commit=True) as session:
|
||||
results = session.query(FileGroup, Archive) \
|
||||
.filter(
|
||||
# Get all groups that contain a PDF that have not been indexed.
|
||||
FileGroup.indexed != True,
|
||||
FileGroup.mimetype == 'text/html',
|
||||
).filter(not_(FileGroup.id.in_(list(invalid_archives)))) \
|
||||
.outerjoin(Archive, Archive.file_group_id == FileGroup.id) \
|
||||
.limit(20)
|
||||
|
||||
readability_file, singlefile_file, readability_json_file, readability_txt_file, screenshot_file = \
|
||||
match_archive_files(group)
|
||||
processed = 0
|
||||
for file_group, archive in results:
|
||||
processed += 1
|
||||
|
||||
archive: Archive = archive_records.get(archive_file.path)
|
||||
if not archive:
|
||||
archive = Archive(singlefile_file=singlefile_file)
|
||||
session.add(archive)
|
||||
archive.readability_file = readability_file
|
||||
archive.readability_json_file = readability_json_file
|
||||
archive.readability_txt_file = readability_txt_file
|
||||
archive.screenshot_file = screenshot_file
|
||||
if archive:
|
||||
try:
|
||||
archive_id = archive.id
|
||||
archive.validate()
|
||||
except Exception:
|
||||
# Archive could not be indexed.
|
||||
invalid_archives |= {file_group.id, }
|
||||
|
||||
if archive.readability_file:
|
||||
archive.readability_file.associated = True
|
||||
archive.readability_file.do_stats()
|
||||
if archive.readability_json_file:
|
||||
archive.readability_json_file.associated = True
|
||||
archive.readability_json_file.do_stats()
|
||||
if archive.readability_txt_file:
|
||||
archive.readability_txt_file.associated = True
|
||||
archive.readability_txt_file.do_stats()
|
||||
if archive.screenshot_file:
|
||||
archive.screenshot_file.associated = True
|
||||
archive.screenshot_file.do_stats()
|
||||
logger.error(f'Unable to validate Archive {archive_id}')
|
||||
if PYTEST:
|
||||
raise
|
||||
else:
|
||||
try:
|
||||
model_archive(file_group, session=session)
|
||||
except InvalidArchive:
|
||||
# FileGroup was not an archive group.
|
||||
invalid_archives |= {file_group.id, }
|
||||
|
||||
archive.singlefile_file.model = Archive.__tablename__
|
||||
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
|
||||
file_group.indexed = True
|
||||
|
||||
archive.singlefile_file.do_index()
|
||||
archive.validate()
|
||||
if processed < 20:
|
||||
# Did not reach limit, do not query again.
|
||||
break
|
||||
|
||||
# Remove this group, it will not be processed by other modelers.
|
||||
del groups[stem]
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
@register_after_refresh
|
||||
|
|
@ -124,7 +195,13 @@ def archive_cleanup():
|
|||
deleted = 0
|
||||
for archive in archives:
|
||||
processed += 1
|
||||
if not lib.is_singlefile_file(archive.singlefile_path):
|
||||
singlefile_path = archive.singlefile_path
|
||||
if not singlefile_path:
|
||||
archive.file_group.model = None
|
||||
session.delete(archive)
|
||||
deleted += 1
|
||||
continue
|
||||
if not lib.is_singlefile_file(singlefile_path):
|
||||
# Archive is not a real archive.
|
||||
archive.singlefile_file.model = None
|
||||
session.delete(archive)
|
||||
|
|
@ -142,45 +219,18 @@ def archive_cleanup():
|
|||
session.delete(domain)
|
||||
|
||||
|
||||
@register_indexer('text/html')
|
||||
class ArchiveIndexer(Indexer, ABC):
|
||||
"""Gathers index data from the files associated with an Archive."""
|
||||
def get_title(path):
|
||||
with path.open('rt') as fh:
|
||||
try:
|
||||
return json.load(fh)['title']
|
||||
except Exception:
|
||||
if not PYTEST:
|
||||
logger.warning(f'Archive readability json file exists, but cannot get title. {path}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_title(file: File):
|
||||
singlefile_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
readability_json_path = singlefile_path.with_suffix('.readability.json')
|
||||
if readability_json_path.is_file():
|
||||
with readability_json_path.open('rt') as fh:
|
||||
try:
|
||||
return json.load(fh)['title']
|
||||
except Exception:
|
||||
if not PYTEST:
|
||||
logger.warning(f'Archive readability json file exists, but cannot get title. {file}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_article(file: File):
|
||||
singlefile_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
readability_txt_path = singlefile_path.with_suffix('.readability.txt')
|
||||
if readability_txt_path.is_file():
|
||||
return readability_txt_path.read_text()
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file: File) -> Tuple:
|
||||
"""
|
||||
Index an Archive file and it's associated files. This mimetype matches non-archive (non-singlefile) html files,
|
||||
so this index will return empty if a readability file is found.
|
||||
|
||||
a = title
|
||||
b = <empty>
|
||||
c = <empty>
|
||||
d = readability article
|
||||
"""
|
||||
if file.path.name.endswith('.readability.html'):
|
||||
# Readability files are not the primary Archive file.
|
||||
return None, None, None, None
|
||||
a = cls.get_title(file)
|
||||
d = cls.get_article(file)
|
||||
d = split_lines_by_length(d)
|
||||
return a, None, None, d
|
||||
def get_article(path):
|
||||
if path.is_file():
|
||||
text = path.read_text()
|
||||
text = split_lines_by_length(text)
|
||||
return text
|
||||
|
|
|
|||
|
|
@ -22,9 +22,9 @@ logger = logger.getChild(__name__)
|
|||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
async def get_archive(_: Request, archive_id: int):
|
||||
archive = lib.get_archive(archive_id=archive_id)
|
||||
archive_file = archive.singlefile_file.__json__()
|
||||
history = [i.singlefile_file.__json__() for i in archive.history]
|
||||
return json_response({'file': archive_file, 'history': history})
|
||||
archive_file_group = archive.file_group.__json__()
|
||||
history = [i.file_group.__json__() for i in archive.history]
|
||||
return json_response({'file_group': archive_file_group, 'history': history})
|
||||
|
||||
|
||||
@bp.delete('/<archive_ids:int>')
|
||||
|
|
@ -52,7 +52,7 @@ archive_limit_limiter = api_param_limiter(100)
|
|||
|
||||
@bp.post('/search')
|
||||
@openapi.definition(
|
||||
summary='Search archive contents and titles',
|
||||
summary='A File search with more filtering related to Archives',
|
||||
body=schema.ArchiveSearchRequest,
|
||||
)
|
||||
@openapi.response(HTTPStatus.OK, schema.ArchiveSearchResponse)
|
||||
|
|
@ -64,6 +64,6 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
|
|||
limit = archive_limit_limiter(body.limit)
|
||||
offset = body.offset or 0
|
||||
|
||||
archives, total = lib.archive_search(search_str, domain, limit, offset, body.order_by)
|
||||
ret = dict(archives=archives, totals=dict(archives=total))
|
||||
file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names)
|
||||
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import copy
|
||||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
|
|
@ -9,7 +8,6 @@ import pytz
|
|||
|
||||
from modules.archive.lib import archive_strftime
|
||||
from modules.archive.models import Archive, Domain
|
||||
from wrolpi.files.models import File
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -68,45 +66,17 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
test_session.add(domain)
|
||||
test_session.flush([domain])
|
||||
|
||||
archive = Archive(
|
||||
readability_json_file=File(path=readability_json_path) if readability_json_path else None,
|
||||
readability_file=File(path=readability_path) if readability_path else None,
|
||||
readability_txt_file=File(path=readability_txt_path) if readability_txt_path else None,
|
||||
screenshot_file=File(path=screenshot_path) if screenshot_path else None,
|
||||
singlefile_file=File(path=singlefile_path, model='archive'),
|
||||
title=title,
|
||||
url=url,
|
||||
domain_id=domain.id if domain else None,
|
||||
archive_datetime=archive_datetime,
|
||||
)
|
||||
test_session.add(archive)
|
||||
archive.singlefile_file.do_index()
|
||||
# Only add files that were created.
|
||||
files = (readability_path, readability_json_path, readability_txt_path, screenshot_path, singlefile_path)
|
||||
files = list(filter(None, files))
|
||||
|
||||
archive = Archive.from_paths(test_session, *files)
|
||||
archive.url = url
|
||||
archive.title = title
|
||||
archive.archive_datetime = next(now)
|
||||
archive.domain = domain
|
||||
archive.validate()
|
||||
|
||||
return archive
|
||||
|
||||
return _
|
||||
|
||||
|
||||
SINGLEFILE_CONTENTS = '''<html><!--
|
||||
Page saved with SingleFile
|
||||
url: https://example.com
|
||||
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
|
||||
--><head><meta charset="utf-8">
|
||||
|
||||
<meta name="msapplication-TileColor" content="#2d89ef">
|
||||
<meta name="theme-color" content="#000000">
|
||||
|
||||
<title>the title</title>
|
||||
|
||||
<body>
|
||||
some body contents
|
||||
</body>
|
||||
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def singlefile_contents() -> str:
|
||||
"""Return a short HTML string that contains an example Singlefile file."""
|
||||
return copy.deepcopy(SINGLEFILE_CONTENTS)
|
||||
|
|
|
|||
|
|
@ -7,22 +7,19 @@ import subprocess
|
|||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from itertools import groupby
|
||||
from typing import Iterator, Optional, Tuple, List
|
||||
from typing import Optional, Tuple, List
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
|
||||
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, walk, \
|
||||
aiohttp_post, match_paths_to_suffixes, iterify
|
||||
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post
|
||||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy
|
||||
from wrolpi.files.lib import handle_search_results
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
|
||||
from wrolpi.vars import DOCKERIZED, PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -52,8 +49,6 @@ class ArchiveFiles:
|
|||
|
||||
def get_archive_directory() -> pathlib.Path:
|
||||
archive_directory = get_media_directory() / 'archive'
|
||||
if not archive_directory.is_dir():
|
||||
archive_directory.mkdir()
|
||||
return archive_directory
|
||||
|
||||
|
||||
|
|
@ -66,7 +61,7 @@ def get_domain_directory(url: str) -> pathlib.Path:
|
|||
elif directory.exists():
|
||||
raise FileNotFoundError(f'Domain directory {directory} is already a file')
|
||||
|
||||
directory.mkdir()
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
return directory
|
||||
|
||||
|
||||
|
|
@ -265,45 +260,23 @@ async def do_archive(url: str) -> Archive:
|
|||
fh.write(json.dumps(readability))
|
||||
|
||||
# Create any File models that we can, index them all.
|
||||
singlefile_file = File(path=archive_files.singlefile, model='archive')
|
||||
singlefile_file.do_index()
|
||||
readability_file = None
|
||||
if archive_files.readability:
|
||||
readability_file = File(path=archive_files.readability, associated=True)
|
||||
readability_file.do_index()
|
||||
readability_json_file = None
|
||||
if archive_files.readability_json:
|
||||
readability_json_file = File(path=archive_files.readability_json, associated=True)
|
||||
readability_json_file.do_index()
|
||||
readability_txt_file = None
|
||||
if archive_files.readability_txt:
|
||||
readability_txt_file = File(path=archive_files.readability_txt, associated=True)
|
||||
readability_txt_file.do_index()
|
||||
screenshot_file = None
|
||||
if archive_files.screenshot:
|
||||
screenshot_file = File(path=archive_files.screenshot, associated=True)
|
||||
screenshot_file.do_index()
|
||||
paths = (
|
||||
archive_files.singlefile, archive_files.readability, archive_files.readability_json,
|
||||
archive_files.readability_txt,
|
||||
archive_files.screenshot)
|
||||
paths = list(filter(None, paths))
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
domain = get_or_create_domain(session, url)
|
||||
archive = Archive(
|
||||
title=title,
|
||||
archive_datetime=now(),
|
||||
singlefile_file=singlefile_file,
|
||||
readability_file=readability_file,
|
||||
readability_json_file=readability_json_file,
|
||||
readability_txt_file=readability_txt_file,
|
||||
screenshot_file=screenshot_file,
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
)
|
||||
session.add(archive)
|
||||
archive = Archive.from_paths(session, *paths)
|
||||
archive.archive_datetime = now()
|
||||
archive.url = url
|
||||
archive.domain = get_or_create_domain(session, url)
|
||||
session.flush()
|
||||
|
||||
return archive
|
||||
|
||||
|
||||
def get_or_create_domain(session, url) -> Domain:
|
||||
def get_or_create_domain(session: Session, url) -> Domain:
|
||||
"""
|
||||
Get/create the Domain for this archive.
|
||||
"""
|
||||
|
|
@ -348,132 +321,11 @@ def delete_archives(*archive_ids: List[int]):
|
|||
archive.delete()
|
||||
|
||||
|
||||
def archive_strptime(dt: str) -> datetime:
|
||||
try:
|
||||
return datetime.strptime(dt, '%Y-%m-%d-%H-%M-%S')
|
||||
except ValueError:
|
||||
return datetime.strptime(dt, '%Y-%m-%d %H:%M:%S')
|
||||
|
||||
|
||||
def archive_strftime(dt: datetime) -> str:
|
||||
return dt.strftime('%Y-%m-%d-%H-%M-%S')
|
||||
|
||||
|
||||
def group_archive_files(files: Iterator[pathlib.Path]) -> groupby:
|
||||
"""
|
||||
Group archive files by their timestamp.
|
||||
"""
|
||||
# groupby requires the files to be sorted.
|
||||
files = sorted(files)
|
||||
# Group archive files by their datetime at the beginning of the file.
|
||||
groups = groupby(files, key=lambda i: i.name[:19])
|
||||
for dt, files in groups:
|
||||
try:
|
||||
dt = archive_strptime(dt)
|
||||
except ValueError:
|
||||
logger.info(f'Ignoring invalid archives of {dt=}')
|
||||
continue
|
||||
|
||||
# Sort the files into their respective slots.
|
||||
archive_files = ArchiveFiles()
|
||||
file = None
|
||||
for file in files:
|
||||
name = file.name
|
||||
# TODO remove the -readability matching once migrated.
|
||||
if name.endswith('.readability.html') or name.endswith('-readability.html'):
|
||||
archive_files.readability = file
|
||||
elif name.endswith('.html'):
|
||||
archive_files.singlefile = file
|
||||
elif name.endswith('.png') or name.endswith('.jpg') or name.endswith('.jpeg'):
|
||||
archive_files.screenshot = file
|
||||
elif name.endswith('.readability.json') or name.endswith('-readability.json'):
|
||||
archive_files.readability_json = file
|
||||
elif name.endswith('.readability.txt') or name.endswith('-readability.txt'):
|
||||
archive_files.readability_txt = file
|
||||
|
||||
if not archive_files.singlefile:
|
||||
logger.warning(f'Archive does not have a singlefile html! Ignoring. {file}')
|
||||
continue
|
||||
if not archive_files.readability_json:
|
||||
logger.warning(f'Archive does not have a json file! Ignoring. {file}')
|
||||
continue
|
||||
|
||||
yield dt, archive_files
|
||||
|
||||
|
||||
OLD_ARCHIVE_MATCHER = re.compile(r'\d{4}-\d\d-\d\d (\d\d:){2}\d\d\.\d{6}.*$')
|
||||
|
||||
|
||||
def migrate_archive_files():
|
||||
"""
|
||||
Rename Archive files from "YYYY-mm-dd HH:MM:SS.ZZZZZ.html" to "YYYY-mm-dd-HH-MM-SS_{TITLE}.html" and all their
|
||||
associated files.
|
||||
"""
|
||||
archive_directory = get_archive_directory()
|
||||
|
||||
def _is_archive_file(path: pathlib.Path) -> bool:
|
||||
return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(OLD_ARCHIVE_MATCHER.match(path.name))
|
||||
|
||||
plan = []
|
||||
|
||||
# It is safer to plan the renames before we make them.
|
||||
for domain_directory in filter(lambda i: i.is_dir(), archive_directory.iterdir()):
|
||||
all_archives_files = filter(_is_archive_file, walk(domain_directory))
|
||||
archive_groups = group_archive_files(all_archives_files)
|
||||
for dt, archive_files in archive_groups:
|
||||
archive_files: ArchiveFiles
|
||||
with archive_files.readability_json.open() as fh:
|
||||
title = escape_file_name(json.load(fh).get('title') or 'NA')
|
||||
|
||||
title = title[:50]
|
||||
|
||||
dt = archive_strftime(dt)
|
||||
prefix = f'{dt}_{title}'
|
||||
singlefile_path = domain_directory / f'{prefix}.html'
|
||||
readability_path = domain_directory / f'{prefix}.readability.html'
|
||||
readability_txt_path = domain_directory / f'{prefix}.readability.txt'
|
||||
readability_json_path = domain_directory / f'{prefix}.readability.json'
|
||||
screenshot_path = domain_directory / f'{prefix}.png'
|
||||
|
||||
# Every Archive is required to have these files.
|
||||
plan.append((archive_files.singlefile, singlefile_path))
|
||||
plan.append((archive_files.readability_json, readability_json_path))
|
||||
# These files are optional.
|
||||
if archive_files.readability:
|
||||
plan.append((archive_files.readability, readability_path))
|
||||
if archive_files.readability_txt:
|
||||
plan.append((archive_files.readability_txt, readability_txt_path))
|
||||
if archive_files.screenshot:
|
||||
plan.append((archive_files.screenshot, screenshot_path))
|
||||
|
||||
if not plan:
|
||||
logger.info('Could not find any Archive files to migrate. Was it already performed?')
|
||||
return plan
|
||||
|
||||
# Check that all new files do not exist.
|
||||
for old, new in plan:
|
||||
if new.exists():
|
||||
raise FileExistsError(f'Cannot migrate archive files! {new} already exists!')
|
||||
|
||||
# Finally, move all the files now that its safe.
|
||||
for old, new in plan:
|
||||
old.rename(new)
|
||||
|
||||
return plan
|
||||
|
||||
|
||||
ARCHIVE_MATCHER = re.compile(r'\d{4}(-\d\d){5}_.*$')
|
||||
ARCHIVE_SUFFIXES = {'.txt', '.html', '.json', '.png', '.jpg', '.jpeg'}
|
||||
|
||||
|
||||
def is_archive_file(path: pathlib.Path) -> bool:
|
||||
"""
|
||||
Archive files are expected to start with the following: %Y-%m-%d-%H-%M-%S
|
||||
they must have one of the following suffixes: .txt, .html, .json, .png, .jpg, .jpeg
|
||||
"""
|
||||
return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(ARCHIVE_MATCHER.match(path.name))
|
||||
|
||||
|
||||
SINGLEFILE_HEADER = '''<!--
|
||||
Page saved with SingleFile'''
|
||||
|
||||
|
|
@ -492,7 +344,7 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
|
|||
# A file name matching exactly the Archive format from WROLPi should always be an Archive.
|
||||
return True
|
||||
with path.open('rt') as fh:
|
||||
# Lastly, try to read the header of this HTML file.
|
||||
# Lastly, try to read the header of this HTML file. Read a large part of the head because URLs can be long.
|
||||
header = fh.read(1000)
|
||||
if SINGLEFILE_HEADER in header:
|
||||
return True
|
||||
|
|
@ -514,68 +366,64 @@ def get_domains():
|
|||
|
||||
|
||||
ARCHIVE_ORDERS = {
|
||||
'date': 'a.archive_datetime ASC, LOWER(a.singlefile_path) ASC',
|
||||
'-date': 'a.archive_datetime DESC NULLS LAST, LOWER(a.singlefile_path) DESC',
|
||||
'date': 'a.archive_datetime ASC, LOWER(fg.primary_path) ASC',
|
||||
'-date': 'a.archive_datetime DESC NULLS LAST, LOWER(fg.primary_path) DESC',
|
||||
'rank': '2 DESC, a.archive_datetime DESC',
|
||||
'-rank': '2 ASC, a.archive_datetime ASC',
|
||||
}
|
||||
|
||||
|
||||
def archive_search(search_str: str, domain: str, limit: int, offset: int, order_by: str) -> Tuple[List[Archive], int]:
|
||||
wheres = []
|
||||
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str]) \
|
||||
-> Tuple[List[dict], int]:
|
||||
# Always filter FileGroups to Archives.
|
||||
wheres = ["fg.model = 'archive'"]
|
||||
joins = []
|
||||
|
||||
params = dict(search_str=search_str, offset=int(offset), limit=int(limit))
|
||||
order = '1 DESC'
|
||||
order_by = '1 DESC'
|
||||
|
||||
if search_str:
|
||||
# A search_str was provided by the user, modify the query to filter by it.
|
||||
select_columns = 'f.path, ts_rank(f.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
|
||||
wheres.append('f.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
select_columns = 'ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
|
||||
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
params['search_str'] = search_str
|
||||
join = 'LEFT JOIN file f on f.path = a.singlefile_path'
|
||||
else:
|
||||
# No search_str provided. Get path and total only. Don't join the "file" to speed up query.
|
||||
select_columns = 'a.singlefile_path AS path, COUNT(*) OVER() AS total'
|
||||
join = ''
|
||||
select_columns = 'fg.id, COUNT(*) OVER() AS total'
|
||||
|
||||
if order_by:
|
||||
if order:
|
||||
try:
|
||||
order = ARCHIVE_ORDERS[order_by]
|
||||
order_by = ARCHIVE_ORDERS[order]
|
||||
except KeyError:
|
||||
raise InvalidOrderBy(f'Invalid order by: {order_by}')
|
||||
raise InvalidOrderBy(f'Invalid order by: {order}')
|
||||
|
||||
if tag_names:
|
||||
where_, params_, join_ = tag_names_to_clauses(tag_names)
|
||||
wheres.append(where_)
|
||||
params.update(params_)
|
||||
joins.append(join_)
|
||||
|
||||
if domain:
|
||||
params['domain'] = domain
|
||||
wheres.append('domain_id = (select id from domains where domains.domain = %(domain)s)')
|
||||
wheres.append('a.domain_id = (select id from domains where domains.domain = %(domain)s)')
|
||||
|
||||
select_columns = f", {select_columns}" if select_columns else ""
|
||||
wheres = '\n AND '.join(wheres)
|
||||
where = f'WHERE\n{wheres}' if wheres else ''
|
||||
join = '\n'.join(joins)
|
||||
stmt = f'''
|
||||
SELECT
|
||||
{select_columns}
|
||||
FROM archive a
|
||||
fg.id -- always get `file_group.id` for `handle_file_group_search_results`
|
||||
{select_columns}
|
||||
FROM file_group fg
|
||||
LEFT JOIN archive a ON a.file_group_id = fg.id
|
||||
{join}
|
||||
{where}
|
||||
ORDER BY {order}
|
||||
GROUP BY fg.id, a.archive_datetime
|
||||
ORDER BY {order_by}
|
||||
OFFSET %(offset)s LIMIT %(limit)s
|
||||
'''.strip()
|
||||
logger.debug(stmt, params)
|
||||
|
||||
results, total = handle_search_results(stmt, params)
|
||||
results, total = handle_file_group_search_results(stmt, params)
|
||||
return results, total
|
||||
|
||||
|
||||
match_archive_paths = partial(match_paths_to_suffixes, suffix_groups=(
|
||||
('.readability.html',),
|
||||
('.html',),
|
||||
('.readability.json',),
|
||||
('.readability.txt',),
|
||||
('.jpg', '.jpeg', '.webp', '.png'),
|
||||
))
|
||||
|
||||
|
||||
@iterify(tuple)
|
||||
def match_archive_files(files: List[File]) -> Tuple[File, File, File, File, File]:
|
||||
archive_paths = match_archive_paths([i.path for i in files])
|
||||
for path in archive_paths:
|
||||
yield next((i for i in files if i.path == path), None)
|
||||
|
|
|
|||
|
|
@ -2,19 +2,19 @@ import datetime
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from typing import Generator, Optional, Iterable
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
import pytz
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, Boolean
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
|
||||
from sqlalchemy.orm import relationship, Session, validates
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from wrolpi.common import ModelHelper, Base, logger
|
||||
from wrolpi.dates import TZDateTime
|
||||
from wrolpi.errors import InvalidArchive
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -28,25 +28,13 @@ class Archive(Base, ModelHelper):
|
|||
__tablename__ = 'archive'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
archive_datetime = Column(TZDateTime)
|
||||
url = Column(String)
|
||||
|
||||
domain_id = Column(Integer, ForeignKey('domains.id'))
|
||||
domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id')
|
||||
|
||||
url = Column(String)
|
||||
title = Column(String)
|
||||
archive_datetime = Column(TZDateTime)
|
||||
validated = Column(Boolean)
|
||||
|
||||
# Associated Files.
|
||||
singlefile_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
singlefile_file: File = relationship('File', primaryjoin='Archive.singlefile_path==File.path')
|
||||
readability_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
readability_file: File = relationship('File', primaryjoin='Archive.readability_path==File.path')
|
||||
readability_json_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
readability_json_file: File = relationship('File', primaryjoin='Archive.readability_json_path==File.path')
|
||||
readability_txt_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
readability_txt_file: File = relationship('File', primaryjoin='Archive.readability_txt_path==File.path')
|
||||
screenshot_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
screenshot_file: File = relationship('File', primaryjoin='Archive.screenshot_path==File.path')
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
def __repr__(self):
|
||||
if self.domain:
|
||||
|
|
@ -54,29 +42,11 @@ class Archive(Base, ModelHelper):
|
|||
f'domain={self.domain.domain}>'
|
||||
return f'<Archive id={self.id} url={self.url} singlefile={repr(str(self.singlefile_path))}>'
|
||||
|
||||
def my_paths(self) -> Generator[pathlib.Path, None, None]:
|
||||
if self.singlefile_path:
|
||||
yield self.singlefile_path
|
||||
if self.readability_path:
|
||||
yield self.readability_path
|
||||
if self.readability_json_path:
|
||||
yield self.readability_json_path
|
||||
if self.readability_txt_path:
|
||||
yield self.readability_txt_path
|
||||
if self.screenshot_path:
|
||||
yield self.screenshot_path
|
||||
def my_paths(self, *mimetypes: str) -> List[pathlib.Path]:
|
||||
return self.file_group.my_paths(*mimetypes)
|
||||
|
||||
def my_files(self) -> Generator[File, None, None]:
|
||||
if self.singlefile_file:
|
||||
yield self.singlefile_file
|
||||
if self.readability_file:
|
||||
yield self.readability_file
|
||||
if self.readability_json_file:
|
||||
yield self.readability_json_file
|
||||
if self.readability_txt_file:
|
||||
yield self.readability_txt_file
|
||||
if self.screenshot_file:
|
||||
yield self.screenshot_file
|
||||
def my_files(self, *mimetypes: str) -> List[dict]:
|
||||
return self.file_group.my_files(*mimetypes)
|
||||
|
||||
def unlink(self):
|
||||
"""
|
||||
|
|
@ -95,32 +65,12 @@ class Archive(Base, ModelHelper):
|
|||
|
||||
return self.archive_datetime > other.archive_datetime
|
||||
|
||||
def __json__(self):
|
||||
stem, _ = split_path_stem_and_suffix(self.singlefile_path) if self.singlefile_path else (None, None)
|
||||
d = dict(
|
||||
archive_datetime=self.archive_datetime,
|
||||
domain=self.domain.dict() if self.domain else None,
|
||||
domain_id=self.domain_id,
|
||||
id=self.id,
|
||||
readability_json_path=self.readability_json_path,
|
||||
readability_path=self.readability_path,
|
||||
readability_txt_path=self.readability_txt_path,
|
||||
screenshot_path=self.screenshot_path,
|
||||
singlefile_path=self.singlefile_path,
|
||||
stem=stem,
|
||||
title=self.title,
|
||||
url=self.url,
|
||||
)
|
||||
return d
|
||||
|
||||
def delete(self):
|
||||
self.unlink()
|
||||
|
||||
session = Session.object_session(self)
|
||||
|
||||
for file in self.my_files():
|
||||
session.delete(file)
|
||||
|
||||
session.delete(self.file_group)
|
||||
session.delete(self)
|
||||
|
||||
if self.domain:
|
||||
|
|
@ -139,101 +89,175 @@ class Archive(Base, ModelHelper):
|
|||
).order_by(Archive.archive_datetime))
|
||||
return history
|
||||
|
||||
def read_singlefile_data(self):
|
||||
"""
|
||||
Read the start of the singlefile (if any) and decode the Archive information.
|
||||
"""
|
||||
path = self.singlefile_file.path if self.singlefile_file else self.singlefile_path
|
||||
if not path:
|
||||
# Can't read contents of nothing.
|
||||
raise ValueError('Cannot read singlefile data when this has no files')
|
||||
@property
|
||||
def singlefile_file(self) -> Optional[dict]:
|
||||
from modules.archive.lib import is_singlefile_file
|
||||
files = self.file_group.my_files('text/html')
|
||||
for file in files:
|
||||
if is_singlefile_file(file['path']):
|
||||
return file
|
||||
|
||||
with path.open('rt') as fh:
|
||||
head = fh.read(1000)
|
||||
if 'Page saved with SingleFile' not in head:
|
||||
return
|
||||
try:
|
||||
if match := MATCH_URL.findall(head):
|
||||
self.url = match[0].strip()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
|
||||
try:
|
||||
if match := MATCH_DATE.findall(head):
|
||||
dt = match[0].strip()
|
||||
dt = ' '.join(dt.split(' ')[:5])
|
||||
# SingleFile uses GMT.
|
||||
dt = datetime.datetime.strptime(
|
||||
dt,
|
||||
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
|
||||
).replace(tzinfo=pytz.timezone('GMT'))
|
||||
self.archive_datetime = dt
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
|
||||
@property
|
||||
def singlefile_path(self) -> Optional[pathlib.Path]:
|
||||
if singlefile_file := self.singlefile_file:
|
||||
return singlefile_file['path']
|
||||
|
||||
@property
|
||||
def readability_file(self) -> Optional[dict]:
|
||||
files = self.file_group.my_files('text/html')
|
||||
for file in files:
|
||||
if file['path'].name.endswith('.readability.html'):
|
||||
return file
|
||||
|
||||
@property
|
||||
def readability_path(self) -> Optional[pathlib.Path]:
|
||||
if readability_file := self.readability_file:
|
||||
return readability_file['path']
|
||||
|
||||
@property
|
||||
def readability_txt_file(self) -> Optional[dict]:
|
||||
files = self.file_group.my_files('text/plain', 'text/html')
|
||||
for file in files:
|
||||
if file['path'].name.endswith('.readability.txt'):
|
||||
return file
|
||||
|
||||
@property
|
||||
def readability_txt_path(self) -> Optional[pathlib.Path]:
|
||||
if readability_txt_file := self.readability_txt_file:
|
||||
return readability_txt_file['path']
|
||||
|
||||
@property
|
||||
def readability_json_file(self) -> Optional[dict]:
|
||||
files = self.file_group.my_files()
|
||||
for file in files:
|
||||
if file['path'].name.endswith('.readability.json'):
|
||||
return file
|
||||
|
||||
@property
|
||||
def readability_json_path(self) -> Optional[pathlib.Path]:
|
||||
if readability_json_file := self.readability_json_file:
|
||||
return readability_json_file['path']
|
||||
|
||||
@property
|
||||
def screenshot_file(self) -> Optional[dict]:
|
||||
files = self.file_group.my_files('image/')
|
||||
for file in files:
|
||||
return file
|
||||
|
||||
@property
|
||||
def screenshot_path(self) -> Optional[pathlib.Path]:
|
||||
if screenshot_file := self.screenshot_file:
|
||||
return screenshot_file['path']
|
||||
|
||||
def read_readability_data(self):
|
||||
"""Read the Readability JSON file, apply its contents to this record."""
|
||||
readability_json_file = self.readability_json_file
|
||||
if not readability_json_file or not readability_json_file.path.is_file():
|
||||
path = self.singlefile_file.path if self.singlefile_file else self.singlefile_path
|
||||
logger.warning(f'{path} does not have an info json file')
|
||||
readability_json_path = self.readability_json_path
|
||||
if not readability_json_path:
|
||||
logger.warning(f'{self.singlefile_path} does not have an info json file')
|
||||
return
|
||||
if not readability_json_path.is_file():
|
||||
error = f'Cannot read data from {readability_json_path} because it not exist.'
|
||||
if PYTEST:
|
||||
raise ValueError(error)
|
||||
logger.error(error)
|
||||
return
|
||||
|
||||
try:
|
||||
with readability_json_file.path.open() as fh:
|
||||
with readability_json_path.open() as fh:
|
||||
json_contents = json.load(fh)
|
||||
url = json_contents.get('url')
|
||||
title = json_contents.get('title')
|
||||
except Exception as e:
|
||||
raise InvalidArchive() from e
|
||||
|
||||
self.url = url
|
||||
self.title = title
|
||||
# Readability is most trusted, it should overwrite any previous data.
|
||||
self.url = url or self.url
|
||||
self.file_group.title = title or self.file_group.title
|
||||
|
||||
def read_singlefile_data(self):
|
||||
"""Read the start of the singlefile (if any) and extract any Archive information."""
|
||||
path = self.singlefile_path
|
||||
if not path:
|
||||
# Can't read contents of nothing.
|
||||
raise ValueError('Cannot read singlefile data when this has no files')
|
||||
|
||||
if self.url and self.archive_datetime:
|
||||
# This data has already been read.
|
||||
return
|
||||
|
||||
with path.open('rt') as fh:
|
||||
head = fh.read(1000)
|
||||
if 'Page saved with SingleFile' not in head:
|
||||
logger.error(f'Could not find SingleFile header in {self}')
|
||||
return
|
||||
|
||||
if not self.url:
|
||||
try:
|
||||
if match := MATCH_URL.findall(head):
|
||||
self.url = match[0].strip()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
|
||||
|
||||
if not self.archive_datetime:
|
||||
try:
|
||||
if match := MATCH_DATE.findall(head):
|
||||
dt = match[0].strip()
|
||||
dt = ' '.join(dt.split(' ')[:5])
|
||||
# SingleFile uses GMT.
|
||||
dt = datetime.datetime.strptime(
|
||||
dt,
|
||||
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
|
||||
).replace(tzinfo=pytz.timezone('GMT'))
|
||||
self.archive_datetime = dt
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
|
||||
|
||||
def apply_domain(self):
|
||||
"""Get the domain from the URL."""
|
||||
from modules.archive.lib import get_or_create_domain
|
||||
domain = None
|
||||
if self.url:
|
||||
domain = get_or_create_domain(Session.object_session(self), self.url)
|
||||
session = Session.object_session(self)
|
||||
if not session:
|
||||
raise ValueError('No session found!')
|
||||
domain = get_or_create_domain(session, self.url)
|
||||
# Clear domain if the URL is missing.
|
||||
self.domain_id = domain.id if domain else None
|
||||
|
||||
def apply_title(self):
|
||||
def apply_singlefile_title(self):
|
||||
"""Get the title from the Singlefile, if its missing."""
|
||||
from modules.archive.lib import get_title_from_html
|
||||
if self.singlefile_path and not self.title:
|
||||
self.title = get_title_from_html(self.singlefile_path.read_text())
|
||||
if self.singlefile_path and not self.file_group.title:
|
||||
self.file_group.title = get_title_from_html(self.singlefile_path.read_text())
|
||||
|
||||
def validate(self):
|
||||
"""Fill in any missing data about this Archive from it's files."""
|
||||
try:
|
||||
self.read_readability_data()
|
||||
self.read_singlefile_data()
|
||||
self.apply_domain()
|
||||
self.apply_title()
|
||||
self.validated = True
|
||||
self.apply_singlefile_title()
|
||||
except Exception as e:
|
||||
logger.warning(f'Unable to validate {self}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def find_by_path(path, session: Session) -> Optional[Base]:
|
||||
archive = session.query(Archive).filter_by(singlefile_path=path).one_or_none()
|
||||
def from_paths(session: Session, *paths: pathlib.Path) -> 'Archive':
|
||||
"""Create a new Archive and FileGroup from the provided paths.
|
||||
|
||||
The files will be read and Archive data extracted."""
|
||||
from modules.archive import model_archive
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
archive = model_archive(file_group, session)
|
||||
return archive
|
||||
|
||||
@staticmethod
|
||||
def find_by_paths(paths, session):
|
||||
archives = list(session.query(Archive).filter(Archive.singlefile_path.in_(paths)))
|
||||
return archives
|
||||
|
||||
@property
|
||||
def primary_path(self):
|
||||
return self.singlefile_path
|
||||
|
||||
|
||||
class Domain(Base, ModelHelper):
|
||||
__tablename__ = 'domains' # plural to avoid conflict
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
domain = Column(String)
|
||||
domain = Column(String, nullable=False)
|
||||
directory = Column(MediaPathType)
|
||||
|
||||
archives: InstrumentedList = relationship('Archive', primaryjoin='Archive.domain_id==Domain.id')
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
|
|
@ -30,9 +30,10 @@ class ArchiveSearchRequest:
|
|||
offset: Optional[int] = None
|
||||
limit: Optional[int] = None
|
||||
order_by: Optional[str] = None
|
||||
tag_names: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchiveSearchResponse:
|
||||
videos: List[ArchiveDict]
|
||||
file_groups: List[ArchiveDict]
|
||||
totals: dict
|
||||
|
|
|
|||
|
|
@ -4,11 +4,14 @@ from http import HTTPStatus
|
|||
|
||||
def check_results(test_client, data, ids):
|
||||
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
if not response.status_code == HTTPStatus.OK:
|
||||
raise AssertionError(str(response.json))
|
||||
|
||||
if ids:
|
||||
assert response.json['archives'], f'Expected {len(ids)} archives but did not receive any'
|
||||
assert [i['archive']['id'] for i in response.json['archives']] == ids, \
|
||||
f'{response.json["archives"][0]["archive"]["id"]}..{response.json["archives"]["archive"][19]["id"]}'
|
||||
file_groups = response.json['file_groups']
|
||||
assert file_groups, f'Expected {len(ids)} archives but did not receive any'
|
||||
assert [i['id'] for i in response.json['file_groups']] == ids, \
|
||||
f'{response.json["file_groups"][0]["id"]}..{response.json["file_groups"][-1]["id"]}'
|
||||
|
||||
|
||||
def test_archives_search(test_session, archive_directory, archive_factory, test_client):
|
||||
|
|
@ -87,22 +90,22 @@ def test_archives_search_no_query(test_session, archive_factory, test_client):
|
|||
# All archives are returned when no `search_str` is passed.
|
||||
request, response = test_client.post('/api/archive/search', content='{}')
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
assert [i['archive']['id'] for i in response.json['archives']] == list(range(100, 80, -1))
|
||||
assert response.json['totals']['archives'] == 100
|
||||
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
|
||||
assert response.json['totals']['file_groups'] == 100
|
||||
|
||||
# All archives are from "example.com".
|
||||
data = dict(domain='example.com')
|
||||
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
assert [i['archive']['id'] for i in response.json['archives']] == list(range(100, 80, -1))
|
||||
assert response.json['totals']['archives'] == 100
|
||||
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
|
||||
assert response.json['totals']['file_groups'] == 100
|
||||
|
||||
# No archives are from "example.org".
|
||||
data = dict(domain='example.org')
|
||||
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
assert [i['archive']['id'] for i in response.json['archives']] == []
|
||||
assert response.json['totals']['archives'] == 0
|
||||
assert [i['id'] for i in response.json['file_groups']] == []
|
||||
assert response.json['totals']['file_groups'] == 0
|
||||
|
||||
|
||||
def test_archive_and_domain_crud(test_session, test_client, archive_factory):
|
||||
|
|
@ -121,14 +124,14 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
|
|||
# Archive1 has Archive2 as history.
|
||||
request, response = test_client.get(f'/api/archive/{archive1.id}')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file']['archive']['id'] == archive1.id
|
||||
assert response.json['history'][0]['archive']['id'] == archive2.id
|
||||
assert response.json['file_group']['id'] == archive1.id
|
||||
assert response.json['history'][0]['id'] == archive2.id
|
||||
|
||||
# Archive2 has Archive1 as history.
|
||||
request, response = test_client.get(f'/api/archive/{archive2.id}')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert archive2.id == response.json['file']['archive']['id']
|
||||
assert response.json['history'][0]['archive']['id'] == archive1.id
|
||||
assert archive2.id == response.json['file_group']['id']
|
||||
assert response.json['history'][0]['id'] == archive1.id
|
||||
|
||||
# Only one domain.
|
||||
request, response = test_client.get(f'/api/archive/domains')
|
||||
|
|
@ -150,7 +153,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
|
|||
# Archive2 no longer has history.
|
||||
request, response = test_client.get(f'/api/archive/{archive2.id}')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file']['archive']['id'] == archive2.id
|
||||
assert response.json['file_group']['id'] == archive2.id
|
||||
assert response.json['history'] == []
|
||||
|
||||
# No Archives, no Domains.
|
||||
|
|
|
|||
|
|
@ -1,31 +1,36 @@
|
|||
import json
|
||||
import pathlib
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from modules.archive import lib
|
||||
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, do_archive, get_domains, \
|
||||
group_archive_files, ArchiveFiles
|
||||
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, do_archive, get_domains
|
||||
from modules.archive.models import Archive, Domain
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.root_api import CustomJSONEncoder
|
||||
from wrolpi.test.common import skip_circleci
|
||||
|
||||
|
||||
def make_fake_request_archive(readability=True, screenshot=True, title=True):
|
||||
async def fake_request_archive(_):
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
|
||||
Image.new('RGB', (1, 1), color='grey').save(fh)
|
||||
fh.seek(0)
|
||||
image = fh.read()
|
||||
singlefile = '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
r = dict(
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
textContent='test readability textContent',
|
||||
title='ジにてこちら' if title else None,
|
||||
) if readability else None
|
||||
s = b'screenshot data' if screenshot else None
|
||||
s = image if screenshot else None
|
||||
return singlefile, r, s
|
||||
|
||||
return fake_request_archive
|
||||
|
|
@ -61,13 +66,12 @@ async def test_dict(test_session):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relationships(test_session, test_directory):
|
||||
async def test_relationships(test_session, test_directory, example_singlefile):
|
||||
with get_db_session(commit=True) as session:
|
||||
url = 'https://wrolpi.org:443'
|
||||
domain = get_or_create_domain(session, url)
|
||||
archive = Archive(
|
||||
singlefile_file=File(path=test_directory / 'wrolpi.org:443/foo'),
|
||||
title='bar',
|
||||
file_group=FileGroup.from_paths(test_session, example_singlefile),
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
)
|
||||
|
|
@ -77,6 +81,50 @@ async def test_relationships(test_session, test_directory):
|
|||
assert archive.domain == domain
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_title(test_session, archive_factory, singlefile_contents_factory):
|
||||
"""An Archive's title can be fetched in multiple ways. This tests from most to least reliable."""
|
||||
# Create some test files, delete all records for a fresh refresh.
|
||||
archive_factory(
|
||||
domain='example.com',
|
||||
url='https://example.com/json-url',
|
||||
title='json title',
|
||||
singlefile_contents=singlefile_contents_factory('singlefile title', 'https://example.com/singlefile-url'),
|
||||
)
|
||||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
assert test_session.query(Archive).count() == 1
|
||||
|
||||
archive1: Archive = test_session.query(Archive).one()
|
||||
|
||||
# Readability JSON is trusted first.
|
||||
assert archive1.url == 'https://example.com/json-url'
|
||||
assert archive1.file_group.title == 'json title'
|
||||
|
||||
# Delete JSON file.
|
||||
archive1.readability_json_path.unlink()
|
||||
test_session.delete(archive1)
|
||||
await refresh_files()
|
||||
|
||||
archive1: Archive = test_session.query(Archive).one()
|
||||
assert archive1.url == 'https://example.com/singlefile-url'
|
||||
assert archive1.file_group.title == 'singlefile title'
|
||||
|
||||
# Clear out title and URL from Singlefile.
|
||||
archive1.singlefile_path.write_text(singlefile_contents_factory('', ''))
|
||||
test_session.delete(archive1)
|
||||
await refresh_files()
|
||||
|
||||
# No title/URL could be found.
|
||||
archive1: Archive = test_session.query(Archive).one()
|
||||
assert archive1.url is None
|
||||
assert archive1.file_group.title is None
|
||||
|
||||
|
||||
def test_archive_refresh_deleted_archive(test_client, test_session, archive_directory, archive_factory):
|
||||
"""Archives/Domains should be deleted when archive files are deleted."""
|
||||
archive1 = archive_factory('example.com', 'https://example.com/1')
|
||||
|
|
@ -125,17 +173,17 @@ def test_archive_refresh_deleted_archive(test_client, test_session, archive_dire
|
|||
check_counts(archive_count=0, domain_count=0)
|
||||
|
||||
|
||||
def test_fills_contents_with_refresh(test_client, test_session, archive_factory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_fills_contents_with_refresh(test_client, test_session, archive_factory, singlefile_contents_factory):
|
||||
"""Refreshing archives fills in any missing contents."""
|
||||
archive1 = archive_factory('example.com', 'https://example.com/one')
|
||||
archive2 = archive_factory('example.com', 'https://example.com/one')
|
||||
# Title can be found in archive3's singlefile
|
||||
archive3 = archive_factory('example.org',
|
||||
singlefile_contents='<html> some stuff<title>last title</title> other stuff</html>')
|
||||
archive3 = archive_factory('example.org', singlefile_contents=singlefile_contents_factory('last title'))
|
||||
archive4 = archive_factory('example.org') # has no contents
|
||||
|
||||
# Clear the title, this will be filled in.
|
||||
archive1.title = archive2.title = archive3.title = archive4.title = None
|
||||
archive1.file_group.title = archive2.file_group.title = archive3.file_group.title = archive4.file_group.title = None
|
||||
|
||||
contents_title_archive = [
|
||||
('foo bar qux', 'my archive', archive1),
|
||||
|
|
@ -143,31 +191,31 @@ def test_fills_contents_with_refresh(test_client, test_session, archive_factory)
|
|||
('baz qux qux qux', None, archive3),
|
||||
]
|
||||
for contents, json_title, archive in contents_title_archive:
|
||||
with archive.readability_txt_path.open('wt') as fh:
|
||||
archive: Archive
|
||||
readability_txt_path = archive.singlefile_path.with_suffix('.readability.txt')
|
||||
with readability_txt_path.open('wt') as fh:
|
||||
fh.write(contents)
|
||||
if json_title:
|
||||
archive.readability_json_path.write_text(json.dumps({'title': json_title, 'url': archive.url}))
|
||||
readability_json_path = archive.singlefile_path.with_suffix('.readability.json')
|
||||
readability_json_path.write_text(json.dumps({'title': json_title, 'url': archive.url}))
|
||||
test_session.commit()
|
||||
|
||||
# Contents are empty. Archives are all missing the title.
|
||||
assert not archive1.singlefile_file.d_text
|
||||
assert not archive2.singlefile_file.d_text
|
||||
assert not archive3.singlefile_file.d_text
|
||||
assert not archive4.singlefile_file.d_text
|
||||
assert not archive1.file_group.d_text
|
||||
assert not archive2.file_group.d_text
|
||||
assert not archive3.file_group.d_text
|
||||
assert not archive4.file_group.d_text
|
||||
|
||||
# Fill the contents.
|
||||
test_client.post('/api/files/refresh')
|
||||
await refresh_files()
|
||||
# The archives will be renamed with their title.
|
||||
archive1, archive2, archive3, archive4 = test_session.query(Archive).order_by(Archive.id)
|
||||
assert archive1.singlefile_file.title
|
||||
assert archive2.singlefile_file.title
|
||||
assert archive3.singlefile_file.title
|
||||
assert not archive4.singlefile_file.d_text
|
||||
assert not archive4.file_group.d_text
|
||||
|
||||
# Missing title is filled.
|
||||
assert archive1.title == 'my archive'
|
||||
assert archive2.title == 'other archive'
|
||||
assert archive3.title == 'last title' # from json
|
||||
# Missing titles are filled.
|
||||
assert archive1.file_group.title == 'my archive'
|
||||
assert archive2.file_group.title == 'other archive'
|
||||
assert archive3.file_group.title == 'last title' # from singlefile HTML
|
||||
|
||||
|
||||
def test_delete_archive(test_session, archive_factory):
|
||||
|
|
@ -183,12 +231,12 @@ def test_delete_archive(test_session, archive_factory):
|
|||
assert archive3.my_paths() and all(i.is_file() for i in archive3.my_paths())
|
||||
|
||||
assert test_session.query(Archive).count() == 3
|
||||
assert test_session.query(File).count() == 15
|
||||
assert test_session.query(FileGroup).count() == 3
|
||||
|
||||
# Delete the oldest.
|
||||
delete_archives(archive1.id, archive3.id)
|
||||
assert test_session.query(Archive).count() == 1
|
||||
assert test_session.query(File).count() == 5
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
# Files were deleted.
|
||||
assert archive1.my_paths() and not any(i.is_file() for i in archive1.my_paths())
|
||||
assert archive3.my_paths() and not any(i.is_file() for i in archive3.my_paths())
|
||||
|
|
@ -202,7 +250,7 @@ def test_delete_archive(test_session, archive_factory):
|
|||
assert domain is None
|
||||
|
||||
# All Files were deleted.
|
||||
assert test_session.query(File).count() == 0
|
||||
assert test_session.query(FileGroup).count() == 0
|
||||
assert archive2.my_paths() and not any(i.is_file() for i in archive2.my_paths())
|
||||
|
||||
|
||||
|
|
@ -215,6 +263,10 @@ def test_get_domains(test_session, archive_factory):
|
|||
archive3 = archive_factory('example.org')
|
||||
test_session.commit()
|
||||
|
||||
assert archive1.domain
|
||||
assert archive2.domain
|
||||
assert archive3.domain
|
||||
|
||||
assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org']
|
||||
|
||||
archive2.delete()
|
||||
|
|
@ -242,17 +294,17 @@ async def test_new_archive(test_session, fake_now):
|
|||
assert isinstance(archive1.readability_path, pathlib.Path)
|
||||
assert isinstance(archive1.readability_txt_path, pathlib.Path)
|
||||
assert isinstance(archive1.screenshot_path, pathlib.Path)
|
||||
assert archive1.title == 'ジにてこちら'
|
||||
assert archive1.file_group.title == 'ジにてこちら'
|
||||
assert archive1.url is not None
|
||||
assert archive1.domain is not None
|
||||
|
||||
# The actual files were dumped and read correctly.
|
||||
with open(archive1.singlefile_path) as fh:
|
||||
assert fh.read() == '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
assert fh.read() == '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
with open(archive1.readability_path) as fh:
|
||||
assert fh.read() == '<html>test readability content</html>'
|
||||
with open(archive1.readability_txt_path) as fh:
|
||||
assert fh.read() == '<html>test readability textContent</html>'
|
||||
assert fh.read() == 'test readability textContent'
|
||||
with open(archive1.readability_json_path) as fh:
|
||||
assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
|
||||
|
||||
|
|
@ -267,7 +319,7 @@ async def test_get_title_from_html(test_session, fake_now):
|
|||
fake_now(datetime(2000, 1, 1))
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert archive.title == 'ジにてこちら'
|
||||
assert archive.file_group.title == 'ジにてこちら'
|
||||
|
||||
async def fake_request_archive(_):
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
|
|
@ -281,7 +333,7 @@ async def test_get_title_from_html(test_session, fake_now):
|
|||
fake_now(datetime(2000, 1, 2))
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert archive.title == 'some title'
|
||||
assert archive.file_group.title == 'some title'
|
||||
|
||||
async def fake_request_archive(_):
|
||||
singlefile = '<html></html>'
|
||||
|
|
@ -295,7 +347,7 @@ async def test_get_title_from_html(test_session, fake_now):
|
|||
fake_now(datetime(2000, 1, 3))
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert archive.title is None
|
||||
assert archive.file_group.title is None
|
||||
|
||||
|
||||
@skip_circleci
|
||||
|
|
@ -333,7 +385,7 @@ def test_get_new_archive_files(fake_now):
|
|||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_title_in_filename(test_session, fake_now, test_directory):
|
||||
async def test_title_in_filename(test_session, fake_now, test_directory, image_bytes_factory):
|
||||
"""
|
||||
The Archive files have the title in the path.
|
||||
"""
|
||||
|
|
@ -342,7 +394,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
|
|||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive1 = await do_archive('https://example.com')
|
||||
|
||||
assert archive1.title == 'ジにてこちら'
|
||||
assert archive1.file_group.title == 'ジにてこちら'
|
||||
|
||||
assert str(archive1.singlefile_path.relative_to(test_directory)) == \
|
||||
'archive/example.com/2000-01-01-00-00-00_ジにてこちら.html'
|
||||
|
|
@ -360,9 +412,9 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
|
|||
r = dict(
|
||||
# No Title from Readability.
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
textContent='test readability textContent',
|
||||
)
|
||||
s = b'screenshot data'
|
||||
s = image_bytes_factory()
|
||||
return singlefile, r, s
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
|
|
@ -386,7 +438,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
|
|||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
)
|
||||
s = b'screenshot data'
|
||||
s = image_bytes_factory()
|
||||
return singlefile, r, s
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
|
|
@ -404,7 +456,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory):
|
|||
'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.png'
|
||||
|
||||
|
||||
@skip_circleci
|
||||
# @skip_circleci
|
||||
def test_refresh_archives(test_session, test_directory, test_client, make_files_structure):
|
||||
"""
|
||||
Archives can be found and put in the database. A single Archive will have multiple files.
|
||||
|
|
@ -418,11 +470,11 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
|
|||
|
||||
make_files_structure({
|
||||
# These are all for an individual Archive.
|
||||
'archive/example.com/2021-10-05 16:20:10.346823.html': singlefile_contents, # renames to 2021-10-05-16-20-10_NA
|
||||
'archive/example.com/2021-10-05 16:20:10.346823.png': None,
|
||||
'archive/example.com/2021-10-05 16:20:10.346823-readability.txt': 'article text contents',
|
||||
'archive/example.com/2021-10-05 16:20:10.346823-readability.json': '{"url": "https://example.com"}',
|
||||
'archive/example.com/2021-10-05 16:20:10.346823-readability.html': '<html></html>',
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.png': None,
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents',
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.json': '{"url": "https://example.com"}',
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
|
||||
# These should log an error because they are missing the singlefile path.
|
||||
'archive/example.com/2021-10-05 16:20:10.readability.html': '<html></html>',
|
||||
'archive/example.com/2021-10-05 16:20:10.readability.json': None,
|
||||
|
|
@ -435,6 +487,11 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
|
|||
test_client.post('/api/files/refresh')
|
||||
assert test_session.query(Archive).count() == 1
|
||||
|
||||
# Cause a re-index of the archive.
|
||||
test_session.query(FileGroup).filter_by(
|
||||
full_stem=str(test_directory / 'archive/example.com/2021-10-05-16-20-10_NA')).one().indexed = False
|
||||
test_session.commit()
|
||||
|
||||
# Running the refresh does not result in a new archive.
|
||||
test_client.post('/api/files/refresh')
|
||||
assert test_session.query(Archive).count() == 1
|
||||
|
|
@ -452,11 +509,15 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
|
|||
content = json.dumps({'search_str': 'text', 'model': 'archive'})
|
||||
request, response = test_client.post('/api/files/search', content=content)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['files'], 'No files matched "text"'
|
||||
assert response.json['files'][0]['model'] == 'archive', 'Returned file was not an archive'
|
||||
assert response.json['files'][0].get('path') == 'archive/example.com/2021-10-05-16-20-10_NA.html', \
|
||||
assert response.json['file_groups'], 'No files matched "text"'
|
||||
assert response.json['file_groups'][0]['model'] == 'archive', 'Returned file was not an archive'
|
||||
assert response.json['file_groups'][0]['data'].get('readability_path') == \
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.html',\
|
||||
'Could not find readability html file'
|
||||
assert response.json['file_groups'][0]['data'].get('readability_txt_path') == \
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt',\
|
||||
'Could not find readability text file containing "text"'
|
||||
assert response.json['files'][0]['archive']['archive_datetime'], 'Archive has no datetime'
|
||||
assert response.json['file_groups'][0]['data']['archive_datetime'], 'Archive has no datetime'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -482,18 +543,9 @@ async def test_refresh_archives_index(test_session, test_directory, test_client,
|
|||
|
||||
archive: Archive = test_session.query(Archive).one()
|
||||
assert archive.singlefile_path == singlefile
|
||||
assert archive.singlefile_file.a_text == 'the title'
|
||||
assert archive.singlefile_file.d_text == 'article text contents'
|
||||
|
||||
# The associated files are not indexed.
|
||||
for name in ('screenshot_file', 'readability_file', 'readability_json_file', 'readability_txt_file'):
|
||||
file = getattr(archive, name)
|
||||
assert file.associated is True, 'The associated file is not marked'
|
||||
assert file.indexed is False
|
||||
# file.a_text is the file name.
|
||||
assert not file.b_text
|
||||
assert not file.c_text
|
||||
assert not file.d_text
|
||||
assert archive.file_group.a_text == 'the title'
|
||||
assert archive.file_group.d_text == 'article text contents'
|
||||
assert archive.file_group.indexed is True
|
||||
|
||||
|
||||
def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure):
|
||||
|
|
@ -506,76 +558,24 @@ def test_refresh_archives_invalid_file(test_session, test_directory, test_client
|
|||
'bogus file': None,
|
||||
})
|
||||
test_client.post('/api/files/refresh')
|
||||
assert test_session.query(File).count() == 3 # Three files total.
|
||||
assert test_session.query(FileGroup).count() == 2 # One bogus file, one archive group.
|
||||
assert test_session.query(Archive).count() == 1 # Only one real archive.
|
||||
|
||||
# Create an invalid Archive, it should be removed during refresh.
|
||||
test_session.add(Archive(singlefile_path=bogus_path))
|
||||
# Claim the bogus file as an Archive, this should be removed.
|
||||
bogus_file: File = test_session.query(File).filter_by(path=bogus_path).one()
|
||||
bogus_file: FileGroup = test_session.query(FileGroup).filter_by(primary_path=bogus_path).one()
|
||||
bogus_file.model = 'archive'
|
||||
archive = Archive(file_group=bogus_file)
|
||||
test_session.add(archive)
|
||||
test_session.commit()
|
||||
assert test_session.query(File).count() == 3
|
||||
assert test_session.query(FileGroup).count() == 2
|
||||
assert test_session.query(Archive).count() == 2
|
||||
|
||||
test_client.post('/api/files/refresh')
|
||||
assert test_session.query(File).count() == 3, 'More files in DB than we created'
|
||||
assert test_session.query(FileGroup).count() == 2, 'More files in DB than we created'
|
||||
assert test_session.query(Archive).count() == 1, 'The bogus Archive was not removed'
|
||||
assert bogus_file.model is None, 'Model was not removed'
|
||||
|
||||
|
||||
def test_group_archive_files(test_directory):
|
||||
"""Archive files should be grouped together when they are the same Archive."""
|
||||
files = [
|
||||
pathlib.Path('2021-10-05 16:20:10.346823.html'),
|
||||
pathlib.Path('2021-10-05 16:20:10.346823.readability.json'),
|
||||
pathlib.Path('2000-01-01-00-00-00_Title.html'),
|
||||
pathlib.Path('2000-01-01-00-00-00_Title.readability.json'),
|
||||
pathlib.Path('not an archive'),
|
||||
pathlib.Path('2000-01-01-00-00-01_Missing a singlefile.readability.json'),
|
||||
]
|
||||
|
||||
group1 = ArchiveFiles(
|
||||
singlefile=pathlib.Path('2000-01-01-00-00-00_Title.html'),
|
||||
readability_json=pathlib.Path('2000-01-01-00-00-00_Title.readability.json')
|
||||
)
|
||||
group2 = ArchiveFiles(
|
||||
singlefile=pathlib.Path('2021-10-05 16:20:10.346823.html'),
|
||||
readability_json=pathlib.Path('2021-10-05 16:20:10.346823.readability.json'),
|
||||
)
|
||||
assert list(group_archive_files(files)) == [
|
||||
(datetime(2000, 1, 1, 0, 0, 0), group1),
|
||||
(datetime(2021, 10, 5, 16, 20, 10), group2),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'name,expected', [
|
||||
('foo', False),
|
||||
('2000-01-01-00-00-00_Some Title.html', True),
|
||||
('2000-01-01-00-00-00_Some Title.readability.json', True),
|
||||
('2000-01-01-00-00-00_Some Title.readability.html', True),
|
||||
('2000-01-01-00-00-00_Some Title.readability.txt', True),
|
||||
('2000-01-01-00-00-00_Some Title.png', True),
|
||||
('2000-01-01-00-00-00_Some Title.jpg', True),
|
||||
('2000-01-01-00-00-00_Some Title.jpeg', True),
|
||||
('2000-01-01-00-00-00_Some NA.html', True),
|
||||
('2000-01-01-00-00-00_NA.readability.json', True),
|
||||
('2000-01-01-00-00-00_NA.readability.html', True),
|
||||
('2000-01-01-00-00-00_NA.readability.txt', True),
|
||||
('2000-01-01-00-00-00_NA.png', True),
|
||||
('2000-01-01-00-00-00_NA.jpg', True),
|
||||
('2000-01-01-00-00-00_NA.jpeg', True),
|
||||
('2000-01-01-00-00-00_ジにてこちら.html', True),
|
||||
('2000-01-01-00-00-00no underscore.html', False),
|
||||
]
|
||||
)
|
||||
def test_is_archive_file(name, expected, test_directory):
|
||||
path = test_directory / name
|
||||
path.touch()
|
||||
assert lib.is_archive_file(path) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'name,expected', [
|
||||
('foo', False),
|
||||
|
|
@ -587,69 +587,12 @@ def test_is_archive_file(name, expected, test_directory):
|
|||
('foo.html', True),
|
||||
]
|
||||
)
|
||||
def test_is_singlefile_file(name, expected, make_files_structure, singlefile_contents):
|
||||
def test_is_singlefile_file(name, expected, make_files_structure, singlefile_contents_factory):
|
||||
"""A Singlefile may not be created by WROLPi, an HTML file created by Singlefile can also be an Archive."""
|
||||
path, = make_files_structure({name: singlefile_contents})
|
||||
path, = make_files_structure({name: singlefile_contents_factory()})
|
||||
assert lib.is_singlefile_file(path) == expected
|
||||
|
||||
|
||||
@skip_circleci
|
||||
def test_migrate_archive_files(test_session, archive_directory, archive_factory):
|
||||
"""Test that migration of all old Archive formatted files are migrated."""
|
||||
# Create some updated archives that should be ignored.
|
||||
archive1 = archive_factory('example.com', title='the title')
|
||||
archive2 = archive_factory('example.com')
|
||||
archive3 = archive_factory('example.org')
|
||||
test_session.commit()
|
||||
assert archive1.singlefile_path.is_file()
|
||||
assert archive2.singlefile_path.is_file()
|
||||
assert archive3.singlefile_path.is_file()
|
||||
|
||||
example_com_directory = archive_directory / 'example.com'
|
||||
example_org_directory = archive_directory / 'example.org'
|
||||
|
||||
# Archive4
|
||||
(example_com_directory / '2021-10-05 16:20:10.346823.html').touch()
|
||||
path = example_com_directory / '2021-10-05 16:20:10.346823.readability.json'
|
||||
path.write_text(json.dumps({'title': 'my title'}))
|
||||
# Archive5
|
||||
(example_com_directory / '2000-01-01 00:00:00.000000.html').touch()
|
||||
path = example_com_directory / '2000-01-01 00:00:00.000000.readability.json'
|
||||
path.write_text(json.dumps({'title': 'invalid#*: title'}))
|
||||
# Archive6
|
||||
(example_org_directory / '2000-01-01 00:00:00.000000.html').touch()
|
||||
path = example_org_directory / '2000-01-01 00:00:00.000000.readability.json'
|
||||
path.write_text(json.dumps({'title': 'A' * 500})) # Really long file name.
|
||||
|
||||
# Migration forms a plan, then performs it.
|
||||
plan = lib.migrate_archive_files()
|
||||
for old, new in plan:
|
||||
# All files are moved.
|
||||
assert not old.is_file()
|
||||
assert new.is_file()
|
||||
# Check that only the outdated files are moved.
|
||||
plan = [(i.relative_to(archive_directory), j.relative_to(archive_directory)) for i, j in plan]
|
||||
assert plan == [
|
||||
(pathlib.Path('example.com/2000-01-01 00:00:00.000000.html'),
|
||||
pathlib.Path('example.com/2000-01-01-00-00-00_invalid# title.html')),
|
||||
(pathlib.Path('example.com/2000-01-01 00:00:00.000000.readability.json'),
|
||||
pathlib.Path('example.com/2000-01-01-00-00-00_invalid# title.readability.json')),
|
||||
(pathlib.Path('example.com/2021-10-05 16:20:10.346823.html'),
|
||||
pathlib.Path('example.com/2021-10-05-16-20-10_my title.html')),
|
||||
(pathlib.Path('example.com/2021-10-05 16:20:10.346823.readability.json'),
|
||||
pathlib.Path('example.com/2021-10-05-16-20-10_my title.readability.json')),
|
||||
(pathlib.Path('example.org/2000-01-01 00:00:00.000000.html'),
|
||||
pathlib.Path('example.org/2000-01-01-00-00-00_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.html')),
|
||||
(pathlib.Path('example.org/2000-01-01 00:00:00.000000.readability.json'),
|
||||
pathlib.Path(
|
||||
'example.org/2000-01-01-00-00-00_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.readability.json')),
|
||||
]
|
||||
# New Archive files still exist.
|
||||
assert all(i.is_file() for i in archive1.my_paths())
|
||||
assert all(i.is_file() for i in archive2.my_paths())
|
||||
assert all(i.is_file() for i in archive3.my_paths())
|
||||
|
||||
|
||||
def test_archive_files(test_directory):
|
||||
af = lib.ArchiveFiles(
|
||||
singlefile=test_directory / 'archive/singlefile.html',
|
||||
|
|
@ -687,17 +630,11 @@ async def test_archive_download_index(test_session, test_directory, image_file):
|
|||
assert archive.readability_path and archive.readability_path.exists()
|
||||
assert archive.readability_json_path
|
||||
assert archive.readability_txt_path and archive.readability_txt_path.exists()
|
||||
assert archive.readability_txt_file.d_text == 'the readability', 'Readability text was not indexed'
|
||||
assert archive.title == 'the singlefile', 'Did not get the title from the singlefile'
|
||||
assert archive.file_group.d_text == 'the readability', 'Readability text was not indexed'
|
||||
assert archive.file_group.title == 'the singlefile', 'Did not get the title from the singlefile'
|
||||
assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \
|
||||
'Did not store the screenshot'
|
||||
|
||||
assert not archive.singlefile_file.associated
|
||||
assert archive.readability_txt_file.associated \
|
||||
and archive.readability_json_file.associated \
|
||||
and archive.readability_file.associated \
|
||||
and archive.screenshot_file.associated, 'Archive files must be associated'
|
||||
|
||||
|
||||
def test_archive_history(test_session, archive_factory):
|
||||
"""Archive's can have a history of other archives.
|
||||
|
|
|
|||
|
|
@ -1,163 +1,89 @@
|
|||
import json
|
||||
import pathlib
|
||||
from abc import ABC
|
||||
from typing import List, Dict
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
import asyncio
|
||||
from typing import List, Tuple
|
||||
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.captions import read_captions, extract_captions
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
|
||||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.files.indexers import Indexer, register_indexer
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.common import logger, limit_concurrent, register_modeler, register_after_refresh
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
from .downloader import video_downloader # Import downloaders so they are registered.
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['video_modeler', 'VideoIndexer']
|
||||
__all__ = ['video_modeler']
|
||||
|
||||
|
||||
def find_video_file_in_group(group: List[File]):
|
||||
return next((i for i in group if i.mimetype.startswith('video/') and i.path.suffix != '.part'), None)
|
||||
VIDEO_PROCESSING_LIMIT = 20
|
||||
|
||||
|
||||
@register_modeler
|
||||
def video_modeler(groups: Dict[str, List[File]], session: Session):
|
||||
new_videos = []
|
||||
async def video_modeler():
|
||||
while True:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup, Video).filter(
|
||||
FileGroup.indexed != True,
|
||||
FileGroup.mimetype.like('video/%'),
|
||||
).outerjoin(Video, Video.file_group_id == FileGroup.id) \
|
||||
.limit(VIDEO_PROCESSING_LIMIT)
|
||||
file_groups: List[Tuple[FileGroup, Video]] = list(file_groups)
|
||||
|
||||
# Search all groups for video files.
|
||||
video_files = {stem: video_file for stem, group in groups.items() if
|
||||
(video_file := find_video_file_in_group(group))}
|
||||
if not video_files:
|
||||
# No videos in these groups.
|
||||
return
|
||||
# Get all matching Video records (if any) in one query.
|
||||
video_paths = [i.path for i in video_files.values()]
|
||||
video_records = {i.video_path: i for i in session.query(Video).filter(Video.video_path.in_(video_paths))}
|
||||
processed = 0
|
||||
for file_group, video in file_groups:
|
||||
processed += 1
|
||||
|
||||
for stem, video_file in video_files.items():
|
||||
group = groups[stem]
|
||||
video_id = None
|
||||
try:
|
||||
if not video:
|
||||
video = Video(file_group=file_group)
|
||||
session.add(video)
|
||||
session.flush([video])
|
||||
video_id = video.id
|
||||
video.validate()
|
||||
except Exception as e:
|
||||
if PYTEST:
|
||||
raise
|
||||
logger.error(f'Unable to model Video {video_id=}', exc_info=e)
|
||||
|
||||
session.flush(group)
|
||||
poster_file = info_json_file = caption_file = None
|
||||
for file in group:
|
||||
if file.mimetype.startswith('image/'):
|
||||
poster_file = file
|
||||
elif file.path.name.endswith('.info.json'):
|
||||
info_json_file = file
|
||||
# Prefer WebVTT over SRT. (SRT cannot be displayed for HTML video).
|
||||
elif file.path.name.endswith('.en.vtt'):
|
||||
caption_file = file
|
||||
elif file.path.name.endswith('.en.srt'):
|
||||
caption_file = caption_file or file
|
||||
file_group.indexed = True
|
||||
|
||||
if poster_file:
|
||||
poster_file.associated = True
|
||||
poster_file.do_stats()
|
||||
if caption_file:
|
||||
caption_file.associated = True
|
||||
caption_file.do_stats()
|
||||
if info_json_file:
|
||||
info_json_file.associated = True
|
||||
info_json_file.do_stats()
|
||||
if processed < VIDEO_PROCESSING_LIMIT:
|
||||
# Did not reach limit, do not query again.
|
||||
break
|
||||
|
||||
video: Video = video_records.get(video_file.path)
|
||||
if not video:
|
||||
video = Video(video_file=video_file)
|
||||
session.add(video)
|
||||
|
||||
size = video_file.path.stat().st_size
|
||||
|
||||
if poster_file != video.poster_file or \
|
||||
caption_file != video.caption_file or \
|
||||
info_json_file != video.info_json_file or \
|
||||
video.size != size:
|
||||
# Files might have been changed. Re-index.
|
||||
video.video_file.indexed = False
|
||||
|
||||
video.poster_file = poster_file
|
||||
video.caption_file = caption_file
|
||||
video.info_json_file = info_json_file
|
||||
video.size = size
|
||||
video_file.model = Video.__tablename__
|
||||
|
||||
new_videos.append(video)
|
||||
|
||||
if new_videos:
|
||||
session.flush(new_videos)
|
||||
for video in new_videos:
|
||||
video.video_file.do_index()
|
||||
video.validate(session)
|
||||
|
||||
# Remove this group, it will not be processed by other modelers.
|
||||
stem, _ = split_path_stem_and_suffix(video.video_path)
|
||||
del groups[stem]
|
||||
logger.debug(f'Modeled {processed} videos')
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
@register_after_refresh
|
||||
@limit_concurrent(1)
|
||||
def video_cleanup():
|
||||
# Claim all Videos in a Channel's directory for that Channel. But, only if they have not yet been claimed.
|
||||
logger.info('Claiming Videos for their Channels')
|
||||
with get_db_curs(commit=True) as curs:
|
||||
# Delete all Videos if the FileModel no longer contains a video.
|
||||
curs.execute('''
|
||||
WITH deleted AS
|
||||
(UPDATE file_group SET model=null WHERE model='video' AND mimetype NOT LIKE 'video/%' RETURNING id)
|
||||
DELETE FROM video WHERE file_group_id = ANY(select id from deleted)
|
||||
''')
|
||||
# Claim all Videos in a Channel's directory for that Channel. But, only if they have not yet been claimed.
|
||||
curs.execute('''
|
||||
UPDATE video v
|
||||
SET channel_id = c.id
|
||||
SET
|
||||
channel_id = c.id
|
||||
FROM channel c
|
||||
LEFT JOIN file_group fg ON fg.primary_path LIKE c.directory || '/%'::VARCHAR
|
||||
WHERE
|
||||
v.video_path LIKE c.directory || '/%'::VARCHAR
|
||||
AND v.channel_id IS NULL
|
||||
v.channel_id IS NULL
|
||||
AND fg.id = v.file_group_id
|
||||
''')
|
||||
curs.execute('''
|
||||
UPDATE file_group fg
|
||||
SET
|
||||
-- Update channel information in file_group.data
|
||||
-- Example: data = {'channel': {'id': 1, 'name': 'example'}}
|
||||
data = fg.data::jsonb || jsonb_build_object('channel',
|
||||
jsonb_build_object('id', v.channel_id, 'name', c.name))
|
||||
FROM video v
|
||||
LEFT JOIN channel c on c.id = v.channel_id
|
||||
WHERE fg.id = v.file_group_id
|
||||
''')
|
||||
|
||||
|
||||
EXTRACT_SUBTITLES = False
|
||||
|
||||
|
||||
@register_indexer('video')
|
||||
class VideoIndexer(Indexer, ABC):
|
||||
"""Handles video files like mp4/ogg."""
|
||||
|
||||
@staticmethod
|
||||
def get_description(file):
|
||||
video_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
info_json_path = video_path.with_suffix('.info.json')
|
||||
if info_json_path.is_file():
|
||||
with info_json_path.open('rt') as fh:
|
||||
try:
|
||||
return json.load(fh)['description']
|
||||
except Exception:
|
||||
if not PYTEST:
|
||||
logger.warning(f'Video info json file exists, but cannot get description. {file}')
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file):
|
||||
"""
|
||||
Index the video file and it's associated files.
|
||||
|
||||
a = title.
|
||||
b = <empty>
|
||||
c = description from info json.
|
||||
d = captions from info json, or extracted from video file.
|
||||
"""
|
||||
a = cls.get_title(file)
|
||||
c = cls.get_description(file)
|
||||
|
||||
# Detect the associated caption files.
|
||||
d = None
|
||||
if (en_vtt := file.path.with_suffix('.en.vtt')).is_file():
|
||||
d = read_captions(en_vtt)
|
||||
elif (vtt := file.path.with_suffix('.vtt')).is_file():
|
||||
d = read_captions(vtt)
|
||||
elif (en_srt := file.path.with_suffix('.en.srt')).is_file():
|
||||
d = read_captions(en_srt)
|
||||
elif (srt := file.path.with_suffix('.srt')).is_file():
|
||||
d = read_captions(srt)
|
||||
elif EXTRACT_SUBTITLES:
|
||||
d = extract_captions(file.path) or ''
|
||||
|
||||
d = split_lines_by_length(d)
|
||||
return a, None, c, d
|
||||
|
|
|
|||
|
|
@ -36,18 +36,16 @@ def download(_, channel_id: int = None):
|
|||
return response.empty()
|
||||
|
||||
|
||||
@content_bp.post('/favorite')
|
||||
@content_bp.post('/tag')
|
||||
@openapi.definition(
|
||||
description='Toggle the favorite flag on a video',
|
||||
body=schema.FavoriteRequest,
|
||||
description='Associate a Video with a Tag',
|
||||
body=schema.TagRequest,
|
||||
)
|
||||
@validate(schema.FavoriteRequest)
|
||||
@openapi.response(HTTPStatus.OK, schema.FavoriteResponse)
|
||||
@validate(schema.TagRequest)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
async def favorite(_: Request, body: schema.FavoriteRequest):
|
||||
_favorite = video_lib.set_video_favorite(body.video_id, body.favorite)
|
||||
ret = {'video_id': body.video_id, 'favorite': _favorite}
|
||||
return json_response(ret, HTTPStatus.OK)
|
||||
async def tag(_: Request, body: schema.TagRequest):
|
||||
video_lib.tag_video(body.video_id, body.tag_name)
|
||||
return response.empty()
|
||||
|
||||
|
||||
@content_bp.get('/statistics')
|
||||
|
|
|
|||
|
|
@ -58,8 +58,6 @@ async def get_channels_video_count() -> Dict[int, int]:
|
|||
FROM
|
||||
channel AS c
|
||||
LEFT JOIN video AS v ON v.channel_id = c.id
|
||||
WHERE
|
||||
v.video_path IS NOT NULL
|
||||
GROUP BY 1
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
|
|
|
|||
|
|
@ -28,10 +28,10 @@ def test_get_channels(test_directory, channel_factory, test_client):
|
|||
def test_get_video(test_client, test_session, simple_channel, video_factory):
|
||||
"""Test that you get can information about a video. Test that video file can be gotten."""
|
||||
now_ = now()
|
||||
video1 = video_factory(channel_id=simple_channel.id)
|
||||
video1.upload_date, video1.title = now_, 'vid1'
|
||||
video2 = video_factory(channel_id=simple_channel.id)
|
||||
video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2'
|
||||
video1 = video_factory(channel_id=simple_channel.id, title='vid1')
|
||||
video1.upload_date = now_
|
||||
video2 = video_factory(channel_id=simple_channel.id, title='vid2')
|
||||
video2.upload_date = now_ + timedelta(seconds=1)
|
||||
test_session.commit()
|
||||
|
||||
# Test that a 404 is returned when no video exists
|
||||
|
|
@ -42,11 +42,11 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
|
|||
# Get the video info we inserted
|
||||
_, response = test_client.get('/api/videos/video/1')
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
assert_dict_contains(response.json['file']['video'], {'title': 'vid1'})
|
||||
assert_dict_contains(response.json['file_group'], {'title': 'vid1'})
|
||||
|
||||
# The next video is included.
|
||||
assert response.json['prev'] is None
|
||||
assert_dict_contains(response.json['next']['video'], {'title': 'vid2'})
|
||||
assert_dict_contains(response.json['next'], {'title': 'vid2'})
|
||||
|
||||
|
||||
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
|
||||
|
|
@ -173,8 +173,8 @@ def test_video_file_name(test_session, simple_video, test_client):
|
|||
"""If a Video has no title, the front-end can use the file name as the title."""
|
||||
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
assert resp.json['file']['video']['video_path'] == 'simple_video.mp4'
|
||||
assert resp.json['file']['video'].get('stem') == 'simple_video'
|
||||
assert resp.json['file_group']['video']['video_path'] == 'simple_video.mp4'
|
||||
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
|
||||
|
||||
|
||||
def test_channel_conflicts(test_client, test_session, test_directory):
|
||||
|
|
@ -443,7 +443,7 @@ def test_search_videos_channel(test_client, test_session, video_factory):
|
|||
d = dict(channel_id=channel1.id)
|
||||
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['files']) == 0
|
||||
assert len(response.json['file_groups']) == 0
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
vid1 = video_factory(title='vid1', channel_id=channel2.id)
|
||||
|
|
@ -454,15 +454,15 @@ def test_search_videos_channel(test_client, test_session, video_factory):
|
|||
# Videos are gotten by their respective channels
|
||||
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['files']) == 1
|
||||
assert response.json['totals']['files'] == 1
|
||||
assert_dict_contains(response.json['files'][0], dict(path='vid2.mp4', video=dict(channel_id=channel1.id)))
|
||||
assert len(response.json['file_groups']) == 1
|
||||
assert response.json['totals']['file_groups'] == 1
|
||||
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
|
||||
|
||||
d = dict(channel_id=channel2.id)
|
||||
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['files']) == 1
|
||||
assert_dict_contains(response.json['files'][0], dict(path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
||||
assert len(response.json['file_groups']) == 1
|
||||
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
||||
|
||||
|
||||
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):
|
||||
|
|
@ -488,7 +488,7 @@ def test_get_channel_videos_pagination(test_client, test_session, simple_channel
|
|||
last_ids = []
|
||||
for offset, video_count in tests:
|
||||
_, response = assert_video_search(channel_id=simple_channel.id, order_by='id', offset=offset, limit=20)
|
||||
assert len(response.json['files']) == video_count, 'Returned videos does not match'
|
||||
current_ids = [i['video']['id'] for i in response.json['files']]
|
||||
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
|
||||
current_ids = [i['id'] for i in response.json['file_groups']]
|
||||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
last_ids = current_ids
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -92,40 +93,3 @@ def test_channels_no_url(test_session, test_directory):
|
|||
|
||||
save_channels_config()
|
||||
import_channels_config()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory):
|
||||
"""A Video will be assigned to a Channel by its info json first, then to its directory."""
|
||||
channel1 = channel_factory(source_id='channel1')
|
||||
channel2 = channel_factory(source_id='channel2')
|
||||
|
||||
# Put the video in the wrong directory. It should be matched to the info json channel first, then to it's directory
|
||||
channel1.source_id = 'channel source id'
|
||||
vid1 = video_factory(
|
||||
with_video_file=(channel2.directory / 'video1.mp4'),
|
||||
with_info_json={'channel_id': channel1.source_id},
|
||||
)
|
||||
test_session.commit()
|
||||
assert not vid1.channel_id
|
||||
|
||||
with mock.patch('wrolpi.common.after_refresh') as mock_after_refresh:
|
||||
# Channel can be found in `video_cleanup`, prevent that with mock.
|
||||
mock_after_refresh.side_effect = []
|
||||
await refresh_files()
|
||||
|
||||
# Channel in info_json was found during validation.
|
||||
assert vid1.channel_id == channel1.id
|
||||
|
||||
# Refresh using cleanup, info json should be trusted.
|
||||
await refresh_files()
|
||||
assert vid1.channel_id == channel1.id
|
||||
|
||||
# Remove the info json, the Video's Channel will be its directory.
|
||||
vid1.channel_id = None
|
||||
vid1.info_json_path.unlink()
|
||||
test_session.commit()
|
||||
await refresh_files()
|
||||
test_session.refresh(vid1)
|
||||
vid1 = test_session.query(Video).one()
|
||||
assert vid1.channel_id == channel2.id
|
||||
|
|
|
|||
|
|
@ -12,35 +12,3 @@ def test_delete_channel_no_url(test_session, test_client, channel_factory):
|
|||
test_session.commit()
|
||||
|
||||
channel.delete_with_videos()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_refresh_censored(test_session, channel_factory, video_factory):
|
||||
"""Censored videos are discovered when refreshing a channel's directory."""
|
||||
channel = channel_factory()
|
||||
channel.info_json = {
|
||||
'entries': [
|
||||
{'id': 'one', 'view_count': 100},
|
||||
{'id': 'two', 'view_count': 200},
|
||||
# Three was deleted, it should be marked as censored.
|
||||
# {'id': 'three', 'view_count': 300},
|
||||
]
|
||||
}
|
||||
vid1 = video_factory(channel_id=channel.id, source_id='one')
|
||||
vid2 = video_factory(channel_id=channel.id, source_id='two')
|
||||
vid3 = video_factory(channel_id=channel.id, source_id='three')
|
||||
test_session.commit()
|
||||
|
||||
assert channel.refreshed is False
|
||||
assert vid1.censored is False and vid1.view_count is None
|
||||
assert vid2.censored is False and vid1.view_count is None
|
||||
assert vid3.censored is False and vid1.view_count is None
|
||||
|
||||
await channel.refresh_files()
|
||||
|
||||
channel = test_session.query(Channel).one()
|
||||
|
||||
assert channel.refreshed is True
|
||||
assert vid1.censored is False and vid1.view_count == 100
|
||||
assert vid2.censored is False and vid2.view_count == 200
|
||||
assert vid3.censored is True and vid3.view_count is None
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ import subprocess
|
|||
import tempfile
|
||||
from datetime import timedelta
|
||||
from decimal import Decimal
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import PIL
|
||||
from PIL import Image
|
||||
|
|
@ -16,25 +15,16 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from wrolpi.captions import FFMPEG_BIN
|
||||
from wrolpi.cmd import FFPROBE_BIN
|
||||
from wrolpi.common import logger, iterify, get_media_directory, \
|
||||
minimize_dict, match_paths_to_suffixes
|
||||
from wrolpi.common import logger, get_media_directory
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import ChannelNameConflict, ChannelURLConflict, \
|
||||
ChannelDirectoryConflict, ChannelSourceIdConflict
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
|
||||
from .models import Channel
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
REQUIRED_OPTIONS = ['name', 'directory']
|
||||
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count'}
|
||||
MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'}
|
||||
MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size',
|
||||
'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id',
|
||||
'view_count'}
|
||||
# These are the supported video formats. These are in order of their preference.
|
||||
VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv')
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
|
|
@ -145,7 +135,7 @@ def generate_video_poster(video_path: Path) -> Tuple[Path, Optional[int]]:
|
|||
|
||||
def convert_image(existing_path: Path, destination_path: Path, ext: str = 'jpeg'):
|
||||
"""Convert an image from one format to another. Remove the existing image file. This will safely overwrite an
|
||||
image if the existing path is the same as the destination path.
|
||||
image if the destination file already exists.
|
||||
"""
|
||||
with tempfile.NamedTemporaryFile(dir=destination_path.parent, delete=False) as fh:
|
||||
img = Image.open(existing_path).convert('RGB')
|
||||
|
|
@ -225,11 +215,8 @@ def check_for_video_corruption(video_path: Path) -> bool:
|
|||
return corrupt
|
||||
|
||||
|
||||
def apply_info_json(channel_id: int):
|
||||
"""Update view_count for all Videos in a channel using its info_json file.
|
||||
|
||||
Mark any videos not in the info_json as "censored".
|
||||
"""
|
||||
def update_view_counts(channel_id: int):
|
||||
"""Update view_count for all Videos in a channel using its info_json file."""
|
||||
with get_db_session() as session:
|
||||
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
|
||||
channel_name = channel.name
|
||||
|
|
@ -255,47 +242,3 @@ def apply_info_json(channel_id: int):
|
|||
curs.execute(stmt, (view_counts_str, channel_id))
|
||||
count = len(curs.fetchall())
|
||||
logger.debug(f'Updated {count} view counts in DB for {channel_name}.')
|
||||
|
||||
# Mark any video not in the info_json entries as censored.
|
||||
source_ids = [i['id'] for i in info['entries']]
|
||||
stmt = '''
|
||||
UPDATE video SET censored=(source_id != ALL(%s))
|
||||
WHERE channel_id=%s
|
||||
'''
|
||||
curs.execute(stmt, (source_ids, channel_id))
|
||||
|
||||
|
||||
minimize_channel = partial(minimize_dict, keys=MINIMUM_CHANNEL_KEYS)
|
||||
minimize_video_info_json = partial(minimize_dict, keys=MINIMUM_INFO_JSON_KEYS)
|
||||
_minimize_video = partial(minimize_dict, keys=MINIMUM_VIDEO_KEYS)
|
||||
|
||||
|
||||
def minimize_video(video: dict) -> dict:
|
||||
"""
|
||||
Return a Video dictionary that contains only the key/values typically used. Minimize the Channel and info_json,
|
||||
if they are present.
|
||||
"""
|
||||
video = _minimize_video(video)
|
||||
|
||||
if video.get('channel'):
|
||||
video['channel'] = minimize_channel(video['channel'])
|
||||
if video.get('info_json'):
|
||||
video['info_json'] = minimize_video_info_json(video['info_json'])
|
||||
|
||||
return video
|
||||
|
||||
|
||||
match_video_paths = partial(match_paths_to_suffixes, suffix_groups=(
|
||||
tuple(f'.{i}' for i in VIDEO_EXTENSIONS),
|
||||
('.jpg', '.jpeg', '.webp', '.png'),
|
||||
('.description',),
|
||||
('.en.vtt', '.en.srt'),
|
||||
('.info.json',),
|
||||
))
|
||||
|
||||
|
||||
@iterify(tuple)
|
||||
def match_video_files(files: List[File]) -> Tuple[File, File, File, File, File]:
|
||||
video_paths = match_video_paths([i.path for i in files])
|
||||
for path in video_paths:
|
||||
yield next((i for i in files if i.path == path), None)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from modules.videos.downloader import VideoDownloader, ChannelDownloader
|
|||
from modules.videos.lib import set_test_channels_config, set_test_downloader_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
|
|
@ -71,13 +71,12 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def simple_video(test_session, test_directory, simple_channel) -> Video:
|
||||
def simple_video(test_session, test_directory, simple_channel, video_file) -> Video:
|
||||
"""A Video with an empty video file whose channel is the Simple Channel."""
|
||||
video_path = test_directory / 'simple_video.mp4'
|
||||
video_path.touch()
|
||||
video_file = File(path=video_path, model='video')
|
||||
video = Video(video_file=video_file, channel_id=simple_channel.id)
|
||||
test_session.add(video)
|
||||
video_file.rename(video_path)
|
||||
video = Video.from_paths(test_session, video_path)
|
||||
video.channel = simple_channel
|
||||
test_session.commit()
|
||||
video = test_session.query(Video).filter_by(id=video.id).one()
|
||||
return video
|
||||
|
|
@ -89,7 +88,7 @@ def video_factory(test_session, test_directory):
|
|||
|
||||
def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file=None,
|
||||
with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False,
|
||||
source_id: str = None):
|
||||
source_id: str = None) -> Video:
|
||||
title = title or str(uuid4())
|
||||
|
||||
if with_video_file and isinstance(with_video_file, (pathlib.Path, str)):
|
||||
|
|
@ -104,13 +103,8 @@ def video_factory(test_session, test_directory):
|
|||
(test_directory / 'videos/NO CHANNEL').mkdir(exist_ok=True, parents=True)
|
||||
path = test_directory / f'videos/NO CHANNEL/{title}.mp4'
|
||||
|
||||
video_file = File(path=path, model='video')
|
||||
test_session.add(video_file)
|
||||
|
||||
if with_video_file:
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
|
||||
else:
|
||||
path.touch()
|
||||
# Create a real video file for mimetype.
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
|
||||
|
||||
info_json_path = None
|
||||
if with_info_json:
|
||||
|
|
@ -118,32 +112,25 @@ def video_factory(test_session, test_directory):
|
|||
with_info_json = {'duration': 5} if with_info_json is True else with_info_json
|
||||
info_json_path = path.with_suffix('.info.json')
|
||||
info_json_path.write_text(json.dumps(with_info_json))
|
||||
test_session.add(File(path=info_json_path))
|
||||
|
||||
poster_path = None
|
||||
if with_poster_ext:
|
||||
poster_path = path.with_suffix(f'.{with_poster_ext}')
|
||||
poster_path = path.with_suffix(f'.{with_poster_ext.lstrip(".")}')
|
||||
Image.new('RGB', (25, 25)).save(poster_path)
|
||||
test_session.add(File(path=poster_path))
|
||||
|
||||
caption_path = None
|
||||
if with_caption_file:
|
||||
caption_path = path.with_suffix('.en.vtt')
|
||||
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', caption_path)
|
||||
test_session.add(File(path=caption_path))
|
||||
|
||||
video = Video(
|
||||
video_path=path,
|
||||
title=title,
|
||||
channel_id=channel_id,
|
||||
source_id=source_id or title,
|
||||
info_json_path=info_json_path,
|
||||
poster_path=poster_path,
|
||||
caption_path=caption_path,
|
||||
upload_date=upload_date,
|
||||
)
|
||||
test_session.add(video)
|
||||
video_file.do_index()
|
||||
paths = (path, info_json_path, poster_path, caption_path)
|
||||
paths = list(filter(None, paths))
|
||||
|
||||
video = Video.from_paths(test_session, *paths)
|
||||
video.channel_id = channel_id
|
||||
video.source_id = source_id or title
|
||||
video.upload_date = upload_date
|
||||
video.validate()
|
||||
return video
|
||||
|
||||
return factory
|
||||
|
|
@ -167,9 +154,8 @@ def video_download_manager(test_download_manager) -> DownloadManager:
|
|||
def test_channels_config(test_directory):
|
||||
(test_directory / 'config').mkdir(exist_ok=True)
|
||||
config_path = test_directory / 'config/channels.yaml'
|
||||
set_test_channels_config(True)
|
||||
yield config_path
|
||||
set_test_channels_config(False)
|
||||
with set_test_channels_config():
|
||||
yield config_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -213,14 +199,19 @@ def assert_video_ids(test_session):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def video_with_search_factory(test_session, test_directory):
|
||||
def video_with_search_factory(test_session, test_directory, video_file_factory):
|
||||
"""A factory that creates a Video record with an associated video File record."""
|
||||
|
||||
def video_with_search(path: str = None, title: str = None, b_text: str = None, c_text: str = None,
|
||||
d_text: str = None, mimetype: str = 'video/mp4'):
|
||||
path = path or (test_directory / str(uuid4())).absolute()
|
||||
video_file = File(path=path, a_text=title, d_text=d_text, mimetype=mimetype, model='video')
|
||||
video = Video(title=title, video_file=video_file)
|
||||
video_file_group = FileGroup.from_paths(test_session, video_file_factory(path))
|
||||
video_file_group.a_text = title
|
||||
video_file_group.b_text = b_text
|
||||
video_file_group.c_text = c_text
|
||||
video_file_group.d_text = d_text
|
||||
video_file_group.mimetype = mimetype
|
||||
video_file_group.model = 'video'
|
||||
video = Video(file_group=video_file_group)
|
||||
test_session.add(video)
|
||||
return video
|
||||
|
||||
|
|
@ -238,7 +229,7 @@ def assert_video_search(test_client):
|
|||
assert_ids: List[int] = None,
|
||||
assert_paths: List[str] = None,
|
||||
search_str: str = None,
|
||||
filters: List[str] = None,
|
||||
tag_names: List[str] = None,
|
||||
offset: int = None,
|
||||
limit: int = None,
|
||||
order_by: str = None,
|
||||
|
|
@ -247,8 +238,8 @@ def assert_video_search(test_client):
|
|||
content = dict()
|
||||
if search_str is not None:
|
||||
content['search_str'] = search_str
|
||||
if filters is not None:
|
||||
content['filters'] = filters
|
||||
if tag_names is not None:
|
||||
content['tag_names'] = tag_names
|
||||
if offset is not None:
|
||||
content['offset'] = offset
|
||||
if limit is not None:
|
||||
|
|
@ -264,16 +255,15 @@ def assert_video_search(test_client):
|
|||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
if assert_ids is not None:
|
||||
assert 'files' in response.json, 'No video files in response'
|
||||
assert [i['video']['id'] for i in response.json['files']] == assert_ids, 'Video IDs do not match'
|
||||
assert 'file_groups' in response.json, 'No video file_groups in response'
|
||||
assert [i['id'] for i in response.json['file_groups']] == assert_ids, 'Video IDs do not match'
|
||||
|
||||
if assert_paths is not None:
|
||||
assert 'files' in response.json, 'No video files in response'
|
||||
assert [i['video']['video_path'] for i in response.json['files']] == assert_paths, \
|
||||
'Video paths do not match'
|
||||
assert 'file_groups' in response.json, 'No video file_groups in response'
|
||||
assert [i['primary_path'] for i in response.json['file_groups']] == assert_paths, 'Video paths do not match'
|
||||
|
||||
if assert_total:
|
||||
assert response.json['totals']['files'] == int(assert_total), 'Total video files does not match'
|
||||
if assert_total and not (count := response.json['totals']['file_groups']) == int(assert_total):
|
||||
raise AssertionError(f'Video count does not match: {count} != {assert_total}')
|
||||
|
||||
return request, response
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import re
|
|||
import traceback
|
||||
from abc import ABC
|
||||
from typing import Tuple, List, Optional, Dict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp.utils
|
||||
from sqlalchemy.orm import Session
|
||||
|
|
@ -15,18 +16,18 @@ from yt_dlp.utils import UnsupportedError, DownloadError
|
|||
from wrolpi.cmd import YT_DLP_BIN
|
||||
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import apply_info_json, get_no_channel_directory, get_videos_directory
|
||||
from .common import get_no_channel_directory, get_videos_directory, update_view_counts
|
||||
from .lib import get_downloader_config
|
||||
from .models import Video, Channel
|
||||
from .schema import ChannelPostRequest
|
||||
from .video_url_resolver import normalize_youtube_shorts_url
|
||||
from .video_url_resolver import normalize_youtube_shorts_url, video_url_resolver
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
ydl_logger = logger.getChild('youtube-dl')
|
||||
|
|
@ -319,13 +320,11 @@ class VideoDownloader(Downloader, ABC):
|
|||
)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
video_file = File.upsert(video_path, session)
|
||||
video: Video = Video.upsert(video_file, session)
|
||||
# Find any files downloaded with the video (poster, caption, etc.).
|
||||
video_paths = glob_shared_stem(video_path)
|
||||
video = Video.from_paths(session, *video_paths)
|
||||
video.source_id = entry['id']
|
||||
video.channel_id = channel_id
|
||||
video.video_file.do_index()
|
||||
video.validate(session)
|
||||
session.flush([video])
|
||||
video_id = video.id
|
||||
except UnrecoverableDownloadError:
|
||||
raise
|
||||
|
|
@ -464,9 +463,6 @@ def update_channel_catalog(channel: Channel, info: dict):
|
|||
logger.warning(f'entries: {entries}')
|
||||
raise KeyError('No id key for entry!') from e
|
||||
|
||||
# In order to store the Video's URL, we will need a quick lookup.
|
||||
urls = {i['id']: i.get('webpage_url') for i in entries}
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Get the channel in this new context.
|
||||
channel: Channel = session.query(Channel).filter_by(id=channel.id).one()
|
||||
|
|
@ -474,20 +470,7 @@ def update_channel_catalog(channel: Channel, info: dict):
|
|||
channel.info_json = info
|
||||
channel.info_date = now()
|
||||
channel.source_id = info.get('id')
|
||||
|
||||
with get_db_curs() as curs:
|
||||
# Get all known videos in this channel.
|
||||
query = 'SELECT source_id FROM video WHERE channel_id=%s AND source_id IS NOT NULL'
|
||||
curs.execute(query, (channel.id,))
|
||||
known_source_ids = {i[0] for i in curs.fetchall()}
|
||||
|
||||
new_source_ids = all_source_ids.difference(known_source_ids)
|
||||
|
||||
logger.info(f'Got {len(new_source_ids)} new videos for channel {channel.name}')
|
||||
channel_id = channel.id
|
||||
for source_id in new_source_ids:
|
||||
url = urls.get(source_id)
|
||||
session.add(Video(source_id=source_id, channel_id=channel_id, url=url))
|
||||
|
||||
# Write the Channel's info to a JSON file.
|
||||
if channel.directory:
|
||||
|
|
@ -496,30 +479,44 @@ def update_channel_catalog(channel: Channel, info: dict):
|
|||
json.dump(info, fh, indent=2)
|
||||
|
||||
# Update all view counts using the latest from the Channel's info_json.
|
||||
apply_info_json(channel_id)
|
||||
update_view_counts(channel_id)
|
||||
|
||||
|
||||
def _find_all_missing_videos(channel_id: id) -> List[Tuple]:
|
||||
"""Get all Video entries which don't have the required media files (i.e. hasn't been downloaded)."""
|
||||
with get_db_curs() as curs:
|
||||
query = f'''
|
||||
SELECT
|
||||
video.id, video.source_id, video.channel_id
|
||||
FROM
|
||||
video
|
||||
LEFT JOIN channel ON channel.id = video.channel_id
|
||||
WHERE
|
||||
channel.url IS NOT NULL
|
||||
AND channel.url != ''
|
||||
AND video.source_id IS NOT NULL
|
||||
AND channel_id = %s
|
||||
AND video.channel_id IS NOT NULL
|
||||
AND (video_path IS NULL OR video_path = '' OR poster_path IS NULL OR poster_path = '')
|
||||
'''
|
||||
params = (channel_id,)
|
||||
curs.execute(query, params)
|
||||
missing_videos = list(curs.fetchall())
|
||||
return missing_videos
|
||||
with get_db_session() as session:
|
||||
channel: Channel = Channel.get_by_id(channel_id)
|
||||
if not channel:
|
||||
raise UnknownChannel(f'Cannot find channel with id {channel_id}')
|
||||
|
||||
def resolve_url(entry: dict) -> str:
|
||||
parsed = urlparse(entry.get('webpage_url') or entry['url'])
|
||||
domain = parsed.hostname
|
||||
url_ = video_url_resolver(domain, entry)
|
||||
url_ = normalize_youtube_shorts_url(url_)
|
||||
return url_
|
||||
|
||||
info_json = channel.info_json
|
||||
# (source_id, url, channel_id)
|
||||
channel_videos = [(i['id'], resolve_url(i), channel_id) for i in info_json['entries']]
|
||||
|
||||
video_source_ids = [i[0] for i in channel_videos]
|
||||
videos_with_source_id = list(session.query(Video).filter(Video.source_id.in_(video_source_ids)))
|
||||
found_source_ids = set()
|
||||
missing_files = []
|
||||
for video in videos_with_source_id:
|
||||
found_source_ids |= {video.source_id, }
|
||||
if not video.poster_path:
|
||||
missing_files.append(video)
|
||||
elif not video.video_path:
|
||||
missing_files.append(video)
|
||||
|
||||
missing_source_ids = set(video_source_ids) - found_source_ids
|
||||
for source_id in missing_source_ids:
|
||||
yield None, source_id, channel_id
|
||||
|
||||
for video in missing_files:
|
||||
yield video.id, video.source_id, channel_id
|
||||
|
||||
|
||||
async def find_all_missing_videos(channel_id: int = None):
|
||||
|
|
|
|||
|
|
@ -1,20 +1,24 @@
|
|||
import contextlib
|
||||
import datetime
|
||||
import functools
|
||||
import html
|
||||
import pathlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Tuple, Optional, Union, Generator
|
||||
import warnings
|
||||
from typing import Tuple, Optional, Generator
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh
|
||||
from wrolpi.dates import from_timestamp, Seconds
|
||||
from wrolpi.captions import extract_captions
|
||||
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh, limit_concurrent
|
||||
from wrolpi.dates import Seconds
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.errors import UnknownDirectory
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.vars import PYTEST
|
||||
from .common import is_valid_poster, convert_image, \
|
||||
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
|
||||
|
|
@ -61,7 +65,10 @@ def get_channel_id_by_source_id(session: Session, source_id: str) -> Optional[in
|
|||
return channel.id if channel else None
|
||||
|
||||
|
||||
def validate_video(video: Video, channel_generate_poster: bool, session: Session):
|
||||
EXTRACT_SUBTITLES = False
|
||||
|
||||
|
||||
def validate_video(video: Video, channel_generate_poster: bool):
|
||||
"""
|
||||
Validate a single video.
|
||||
|
||||
|
|
@ -71,13 +78,13 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
|
|||
A Video is also valid when it has a JPEG poster, if any. If no poster can be
|
||||
found, it will be generated from the video file.
|
||||
"""
|
||||
info_json_path = video.info_json_file.path if video.info_json_file else video.info_json_path
|
||||
json_data_missing = bool(video.title) and bool(video.duration) and bool(video.view_count) and bool(video.url) \
|
||||
and bool(video.upload_date)
|
||||
info_json_path = video.info_json_path
|
||||
json_data_missing = bool(video.file_group.title) and bool(video.duration) and bool(video.view_count) \
|
||||
and bool(video.url) and bool(video.upload_date)
|
||||
if info_json_path and json_data_missing is False:
|
||||
# These properties can be found in the info json.
|
||||
video_info_json = process_video_info_json(video)
|
||||
video.title = video_info_json.title
|
||||
video.file_group.title = video_info_json.title
|
||||
video.duration = video_info_json.duration
|
||||
video.url = video_info_json.url
|
||||
# View count will probably be overwritten by more recent data when this Video's Channel is
|
||||
|
|
@ -89,35 +96,40 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
|
|||
|
||||
video_path = video.video_path
|
||||
|
||||
if not video.title or not video.upload_date or not video.source_id:
|
||||
if video_path and (not video.file_group.title or not video.upload_date or not video.source_id):
|
||||
# Video is missing things that can be extracted from the video file name.
|
||||
# These are the least trusted, so anything already on the video should be trusted.
|
||||
_, upload_date, source_id, title = parse_video_file_name(video_path)
|
||||
video.title = video.title or html.unescape(title)
|
||||
video.upload_date = video.upload_date or upload_date
|
||||
upload_date = upload_date or video.upload_date
|
||||
if upload_date:
|
||||
upload_date = datetime.datetime.strptime(upload_date, '%Y%M%d')
|
||||
upload_date = upload_date.replace(tzinfo=datetime.timezone.utc)
|
||||
video.file_group.title = video.file_group.title or html.unescape(title)
|
||||
video.upload_date = upload_date
|
||||
video.source_id = video.source_id or source_id
|
||||
|
||||
if not video.size or not video.modification_datetime:
|
||||
stat = video_path.stat()
|
||||
video.size = video.size or stat.st_size
|
||||
video.modification_datetime = video.modification_datetime or from_timestamp(stat.st_mtime)
|
||||
|
||||
if channel_generate_poster:
|
||||
# Try to convert/generate, but keep the old poster if those fail.
|
||||
new_poster_path, duration = convert_or_generate_poster(video) or video.poster_path
|
||||
if new_poster_path:
|
||||
# Poster was created/updated.
|
||||
poster_file = File.upsert(new_poster_path, session)
|
||||
session.flush([poster_file])
|
||||
video.poster_file = poster_file
|
||||
video.poster_path = new_poster_path
|
||||
video.file_group.append_files(new_poster_path)
|
||||
if duration:
|
||||
video.duration = duration
|
||||
|
||||
if not video.duration:
|
||||
if video_path and not video.duration:
|
||||
# Duration was not retrieved during poster generation.
|
||||
video.duration = get_video_duration(video_path)
|
||||
|
||||
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:
|
||||
# Caption file was deleted, clear out old captions.
|
||||
video.file_group.d_text = None
|
||||
elif video_path and not video.file_group.d_text and EXTRACT_SUBTITLES:
|
||||
# Captions were not found, extract them from the video.
|
||||
video.file_group.d_text = extract_captions(video_path)
|
||||
elif video_path and video.caption_paths and not video.file_group.d_text:
|
||||
video.file_group.d_text = video.get_caption_text()
|
||||
|
||||
|
||||
def convert_or_generate_poster(video: Video) -> Tuple[Optional[pathlib.Path], Optional[int]]:
|
||||
"""
|
||||
|
|
@ -128,28 +140,32 @@ def convert_or_generate_poster(video: Video) -> Tuple[Optional[pathlib.Path], Op
|
|||
"""
|
||||
video_path = video.video_path
|
||||
# Modification/generation of poster is enabled for this channel.
|
||||
if video.poster_path or video.poster_file:
|
||||
if video.poster_path:
|
||||
# Check that the poster is a more universally supported JPEG.
|
||||
old: pathlib.Path = video.poster_path if video.poster_path else video.poster_file.path
|
||||
old: pathlib.Path = video.poster_path
|
||||
new = old.with_suffix('.jpg')
|
||||
|
||||
if old != new and new.exists():
|
||||
# Destination JPEG already exists (it may have the wrong format).
|
||||
old.unlink()
|
||||
old = video.poster_path = new
|
||||
old = new
|
||||
|
||||
if not is_valid_poster(old):
|
||||
# Poster is not valid, convert it and place it in the new location.
|
||||
try:
|
||||
convert_image(old, new)
|
||||
old.unlink(missing_ok=True)
|
||||
if not new.is_file():
|
||||
raise FileNotFoundError(f'Failed to convert poster: {new}')
|
||||
if old != new:
|
||||
# Only remove the old one if we are not converting in-place.
|
||||
old.unlink(missing_ok=True)
|
||||
logger.info(f'Converted invalid poster {repr(str(old))} to {repr(str(new))}')
|
||||
return new, None
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to convert invalid poster {old} to {new}', exc_info=e)
|
||||
return None, None
|
||||
|
||||
if not video.poster_path and not video.poster_file:
|
||||
if not video.poster_path:
|
||||
# Video poster was not discovered, or converted. Let's generate it.
|
||||
try:
|
||||
poster_path, duration = generate_video_poster(video_path)
|
||||
|
|
@ -185,10 +201,12 @@ class ChannelsConfig(ConfigFile):
|
|||
|
||||
@property
|
||||
def favorites(self) -> dict:
|
||||
warnings.warn('Favorites have bee moved to Tags', DeprecationWarning)
|
||||
return self._config['favorites']
|
||||
|
||||
@favorites.setter
|
||||
def favorites(self, value: dict):
|
||||
warnings.warn('Favorites have bee moved to Tags', DeprecationWarning)
|
||||
self.update({'favorites': value})
|
||||
|
||||
|
||||
|
|
@ -205,12 +223,12 @@ def get_channels_config():
|
|||
return CHANNELS_CONFIG
|
||||
|
||||
|
||||
def set_test_channels_config(enable: bool):
|
||||
@contextlib.contextmanager
|
||||
def set_test_channels_config():
|
||||
global TEST_CHANNELS_CONFIG
|
||||
if enable is True:
|
||||
TEST_CHANNELS_CONFIG = ChannelsConfig()
|
||||
else:
|
||||
TEST_CHANNELS_CONFIG = None
|
||||
TEST_CHANNELS_CONFIG = ChannelsConfig()
|
||||
yield
|
||||
TEST_CHANNELS_CONFIG = None
|
||||
|
||||
|
||||
class VideoDownloaderConfig(ConfigFile):
|
||||
|
|
@ -334,46 +352,14 @@ def get_channels_config_from_db(session: Session) -> dict:
|
|||
"""Create a dictionary that contains all the Channels from the DB."""
|
||||
channels = session.query(Channel).order_by(Channel.directory).all()
|
||||
channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory'])
|
||||
|
||||
# Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel
|
||||
# is deleted or the DB is wiped.
|
||||
favorite_videos = session.query(Video).filter(Video.favorite != None, Video.video_path != None).all() # noqa
|
||||
favorites = defaultdict(lambda: {})
|
||||
for video in favorite_videos:
|
||||
if video.channel:
|
||||
directory = video.channel.directory.relative_to(get_media_directory())
|
||||
favorites[str(directory)][video.video_path.name] = dict(favorite=video.favorite)
|
||||
else:
|
||||
favorites['NO CHANNEL'][video.video_path.name] = dict(favorite=video.favorite)
|
||||
favorites = dict(favorites)
|
||||
|
||||
return dict(channels=channels, favorites=favorites)
|
||||
|
||||
|
||||
def _detect_old_favorites(config: Union[ChannelsConfig, dict]) -> bool:
|
||||
# TODO remove these old favorites after beta.
|
||||
media_directory = get_media_directory()
|
||||
favorites = config.favorites if hasattr(config, 'favorites') else config['favorites']
|
||||
for channel, favorites in favorites.items():
|
||||
if channel == 'NO CHANNEL':
|
||||
continue
|
||||
if not (media_directory / channel).is_dir():
|
||||
return True
|
||||
return False
|
||||
return dict(channels=channels)
|
||||
|
||||
|
||||
@optional_session()
|
||||
def save_channels_config(session=None, preserve_favorites: bool = True):
|
||||
def save_channels_config(session: Session = None):
|
||||
"""Get the Channel information from the DB, save it to the config."""
|
||||
config = get_channels_config_from_db(session)
|
||||
channels_config = get_channels_config()
|
||||
# TODO remove these old favorites after beta.
|
||||
old_favorites = _detect_old_favorites(channels_config)
|
||||
if old_favorites and config['favorites']:
|
||||
# There are favorites in the database, remove the old favorites.
|
||||
channels_config.favorites = {}
|
||||
if preserve_favorites:
|
||||
config['favorites'].update(channels_config.favorites or {})
|
||||
channels_config.update(config)
|
||||
|
||||
|
||||
|
|
@ -382,6 +368,7 @@ channel_import_logger = logger.getChild('channel_import')
|
|||
|
||||
@before_startup
|
||||
@register_after_refresh
|
||||
@limit_concurrent(1)
|
||||
def import_channels_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
if PYTEST and not TEST_CHANNELS_CONFIG:
|
||||
|
|
@ -469,15 +456,19 @@ def import_channels_config():
|
|||
else:
|
||||
channel_dir = get_no_channel_directory()
|
||||
|
||||
favorites_tag = Tag.find_by_name('Favorite')
|
||||
if not favorites_tag:
|
||||
channel_import_logger.warning(f'Cannot tag favorite videos without Favorite tag!')
|
||||
return
|
||||
|
||||
# Set favorite Videos of this Channel.
|
||||
for video_path, data in favorites.items():
|
||||
# Favorite in the config is the name of the video_path. Add the channel directory onto this
|
||||
# video_path, so we can match the complete path for the Video.
|
||||
video_path = channel_dir / video_path
|
||||
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
|
||||
# If no Video is found, it may be that we need to refresh.
|
||||
if video:
|
||||
video.favorite = data['favorite']
|
||||
file_group = FileGroup.find_by_path(video_path, session)
|
||||
if file_group:
|
||||
file_group.add_tag(favorites_tag, session)
|
||||
else:
|
||||
channel_import_logger.warning(f'Cannot find video to favorite: {video_path}')
|
||||
except Exception as e:
|
||||
|
|
@ -503,22 +494,12 @@ async def get_statistics():
|
|||
SELECT
|
||||
-- total videos
|
||||
COUNT(id) AS "videos",
|
||||
-- total videos that are marked as favorite
|
||||
COUNT(id) FILTER (WHERE favorite IS NOT NULL) AS "favorites",
|
||||
-- total videos downloaded over the past week/month/year
|
||||
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 week') AS "week",
|
||||
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 month') AS "month",
|
||||
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 year') AS "year",
|
||||
-- sum of all video lengths in seconds
|
||||
COALESCE(SUM(duration), 0) AS "sum_duration",
|
||||
-- sum of all video file sizes
|
||||
COALESCE(SUM(size), 0)::BIGINT AS "sum_size",
|
||||
-- largest video
|
||||
COALESCE(MAX(size), 0) AS "max_size"
|
||||
COUNT(id) FILTER (WHERE upload_date >= current_date - interval '1 year') AS "year"
|
||||
FROM
|
||||
video
|
||||
WHERE
|
||||
video_path IS NOT NULL
|
||||
''')
|
||||
video_stats = dict(curs.fetchone())
|
||||
|
||||
|
|
@ -526,8 +507,7 @@ async def get_statistics():
|
|||
curs.execute('''
|
||||
SELECT
|
||||
DATE_TRUNC('month', months.a),
|
||||
COUNT(id)::BIGINT,
|
||||
SUM(size)::BIGINT AS "size"
|
||||
COUNT(id)::BIGINT
|
||||
FROM
|
||||
generate_series(
|
||||
date_trunc('month', current_date) - interval '2 years',
|
||||
|
|
@ -538,7 +518,6 @@ async def get_statistics():
|
|||
video.upload_date >= date_trunc('month', months.a)
|
||||
AND video.upload_date < date_trunc('month', months.a) + interval '1 month'
|
||||
AND video.upload_date IS NOT NULL
|
||||
AND video.video_path IS NOT NULL
|
||||
GROUP BY
|
||||
1
|
||||
ORDER BY
|
||||
|
|
@ -610,18 +589,11 @@ def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path
|
|||
directory = str(directory).rstrip('/')
|
||||
|
||||
curs.execute(f'''
|
||||
SELECT f1.path
|
||||
FROM (
|
||||
-- Get array of mimetypes for each full_stem group.
|
||||
select f2.full_stem, array_agg(f2.mimetype) as mimetypes
|
||||
from file f2
|
||||
where f2.full_stem like '{directory}/%'
|
||||
group by 1) AS stem_groups
|
||||
-- Get all files that match the full_stem.
|
||||
LEFT JOIN file f1 ON stem_groups.full_stem = f1.full_stem
|
||||
-- Get all groups which DO NOT contain one of the video mimetypes.
|
||||
WHERE NOT (select array_agg(f3.mimetype) from file f3 where f3.mimetype like 'video/%') && stem_groups.mimetypes
|
||||
SELECT files
|
||||
FROM file_group
|
||||
WHERE
|
||||
mimetype NOT LIKE 'video%'
|
||||
AND full_stem LIKE '{directory}/%'
|
||||
''')
|
||||
results = map(lambda i: pathlib.Path(i[0]), curs.fetchall())
|
||||
results = filter(lambda i: i.is_file(), results)
|
||||
results = (pathlib.Path(j['path']) for i in curs.fetchall() for j in i[0])
|
||||
yield from results
|
||||
|
|
|
|||
|
|
@ -1,21 +1,23 @@
|
|||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Generator, Dict, List
|
||||
from typing import Optional, Dict, List, Union
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, BigInteger, any_
|
||||
from sqlalchemy.orm import relationship, Session, deferred
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from wrolpi.captions import read_captions
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task
|
||||
from wrolpi.dates import now, TZDateTime
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.errors import UnknownVideo
|
||||
from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.lib import refresh_files, get_mimetype, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.tags import Tag, TagFile
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -24,119 +26,73 @@ logger = logger.getChild(__name__)
|
|||
class Video(ModelHelper, Base):
|
||||
__tablename__ = 'video'
|
||||
id = Column(Integer, primary_key=True)
|
||||
channel_id = Column(Integer, ForeignKey('channel.id'))
|
||||
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
|
||||
validated = Column(Boolean, default=False)
|
||||
|
||||
censored = Column(Boolean, default=False)
|
||||
duration = Column(Integer)
|
||||
favorite = Column(TZDateTime)
|
||||
modification_datetime = Column(TZDateTime)
|
||||
size = Column(Integer)
|
||||
source_id = Column(String)
|
||||
title = Column(String)
|
||||
upload_date = Column(TZDateTime)
|
||||
upload_date: datetime.datetime = Column(TZDateTime)
|
||||
url = Column(String)
|
||||
view_count = Column(Integer)
|
||||
viewed = Column(TZDateTime)
|
||||
|
||||
# Associated Files
|
||||
caption_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
caption_file: File = relationship('File', primaryjoin='Video.caption_path==File.path')
|
||||
info_json_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
info_json_file: File = relationship('File', primaryjoin='Video.info_json_path==File.path')
|
||||
poster_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
poster_file: File = relationship('File', primaryjoin='Video.poster_path==File.path')
|
||||
video_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
video_file: File = relationship('File', primaryjoin='Video.video_path==File.path')
|
||||
channel_id = Column(Integer, ForeignKey('channel.id'))
|
||||
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
def __repr__(self):
|
||||
v = self.video_file
|
||||
v = None
|
||||
if self.video_path:
|
||||
video_path = self.video_path.path if hasattr(self.video_path, 'path') else self.video_path
|
||||
v = video_path.relative_to(get_media_directory())
|
||||
return f'<Video id={self.id} title={repr(self.title)} path={v} channel={self.channel_id} ' \
|
||||
v = self.video_path.relative_to(get_media_directory())
|
||||
return f'<Video id={self.id} title={repr(self.file_group.name)} path={v} channel={self.channel_id} ' \
|
||||
f'source_id={repr(self.source_id)}>'
|
||||
|
||||
@classmethod
|
||||
def upsert(cls, video_file: File, session: Session):
|
||||
video = session.query(Video).filter_by(video_file=video_file).one_or_none()
|
||||
if not video:
|
||||
video = Video(video_file=video_file)
|
||||
video: Video
|
||||
video.video_path = video.video_path or video_file.path
|
||||
video.video_file = video.video_file or video_file
|
||||
video.size = video.video_path.stat().st_size
|
||||
session.add(video)
|
||||
def __json__(self) -> dict:
|
||||
d = self.file_group.__json__()
|
||||
|
||||
video.find_my_files()
|
||||
channel = None
|
||||
if self.channel:
|
||||
channel = dict(id=self.channel.id, name=self.channel.name)
|
||||
|
||||
# Claim the video file for this model.
|
||||
video.video_file.model = cls.__tablename__
|
||||
|
||||
return video
|
||||
|
||||
def my_paths(self) -> Generator[Path, None, None]:
|
||||
"""Return all paths of this Video. Returns nothing if all paths are None."""
|
||||
if self.poster_path:
|
||||
yield self.poster_path
|
||||
if self.caption_path:
|
||||
yield self.caption_path
|
||||
if self.info_json_path:
|
||||
yield self.info_json_path
|
||||
if self.video_path:
|
||||
yield self.video_path
|
||||
|
||||
def my_files(self) -> Generator[File, None, None]:
|
||||
"""Return all Files of this Video. Returns nothing if all paths are None."""
|
||||
if self.poster_file:
|
||||
yield self.poster_file
|
||||
if self.caption_file:
|
||||
yield self.caption_file
|
||||
if self.info_json_file:
|
||||
yield self.info_json_file
|
||||
if self.video_file:
|
||||
yield self.video_file
|
||||
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
|
||||
d['video'] = dict(
|
||||
caption=self.file_group.d_text,
|
||||
caption_files=self.caption_files,
|
||||
channel=channel,
|
||||
channel_id=self.channel_id,
|
||||
description=self.file_group.c_text or self.get_video_description(),
|
||||
id=self.id,
|
||||
info_json_file=self.info_json_file,
|
||||
info_json_path=self.info_json_path,
|
||||
poster_file=self.poster_file,
|
||||
poster_path=self.poster_path,
|
||||
source_id=self.source_id,
|
||||
stem=split_path_stem_and_suffix(self.video_path)[0],
|
||||
upload_date=self.upload_date.isoformat() if self.upload_date else None,
|
||||
url=self.url,
|
||||
video_path=self.video_path,
|
||||
view_count=self.view_count,
|
||||
)
|
||||
return d
|
||||
|
||||
def delete(self):
|
||||
"""
|
||||
Remove all files and File records related to this video. Delete this Video record.
|
||||
Add it to it's Channel's skip list.
|
||||
"""
|
||||
for path in self.my_paths():
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
needs_save = bool(self.favorite)
|
||||
|
||||
"""Remove all files and File records related to this video. Delete this Video record.
|
||||
Add it to it's Channel's skip list."""
|
||||
self.add_to_skip_list()
|
||||
self.favorite = None
|
||||
session = Session.object_session(self)
|
||||
session.commit()
|
||||
session.delete(self)
|
||||
|
||||
for file in self.my_files():
|
||||
session.delete(file)
|
||||
|
||||
if needs_save:
|
||||
# Save the Channels config. This video was a favorite and that needs to be removed.
|
||||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config(preserve_favorites=False)
|
||||
self.file_group.delete()
|
||||
|
||||
def add_to_skip_list(self):
|
||||
"""Add this video to it's Channel's skip list."""
|
||||
if self.channel and self.source_id:
|
||||
self.channel.add_video_to_skip_list(self.source_id)
|
||||
|
||||
def set_favorite(self, favorite: bool) -> Optional[datetime]:
|
||||
self.favorite = now() if favorite else None
|
||||
return self.favorite
|
||||
|
||||
def set_viewed(self):
|
||||
self.viewed = now()
|
||||
|
||||
def get_info_json(self) -> Optional[Dict]:
|
||||
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
|
||||
info_json_path = self.info_json_file.path if self.info_json_file else self.info_json_path
|
||||
info_json_path = self.info_json_path
|
||||
if not info_json_path:
|
||||
return
|
||||
|
||||
|
|
@ -145,10 +101,6 @@ class Video(ModelHelper, Base):
|
|||
return json.load(fh)
|
||||
except FileNotFoundError:
|
||||
logger.warning(f'Unable to find info json file! {info_json_path}')
|
||||
if self.video_path.is_file():
|
||||
# Clear out the info_json only if the video file exists. We don't want to clear out the info_json if
|
||||
# the drive hasn't been mounted.
|
||||
self.info_json_file = self.info_json_path = None
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f'Unable to parse info json {self.info_json_path}', exc_info=e)
|
||||
|
|
@ -165,11 +117,6 @@ class Video(ModelHelper, Base):
|
|||
if description:
|
||||
return description
|
||||
|
||||
if self.description_path:
|
||||
with open(self.description_path, 'rt') as fh:
|
||||
contents = fh.read()
|
||||
return contents
|
||||
|
||||
def get_surrounding_videos(self):
|
||||
"""
|
||||
Get the previous and next videos around this video. The videos must be in the same channel.
|
||||
|
|
@ -236,96 +183,111 @@ class Video(ModelHelper, Base):
|
|||
|
||||
return previous_video, next_video
|
||||
|
||||
def __json__(self):
|
||||
from modules.videos.common import minimize_video_info_json
|
||||
info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None
|
||||
stem, _ = split_path_stem_and_suffix(self.video_path) if self.video_file else (None, None)
|
||||
|
||||
d = dict(
|
||||
caption_path=self.caption_path,
|
||||
censored=self.censored,
|
||||
channel=self.channel,
|
||||
channel_id=self.channel_id,
|
||||
modification_datetime=self.modification_datetime,
|
||||
duration=self.duration,
|
||||
favorite=self.favorite,
|
||||
id=self.id,
|
||||
info_json=info_json,
|
||||
info_json_path=self.info_json_path,
|
||||
poster_path=self.poster_path,
|
||||
size=self.size,
|
||||
source_id=self.source_id,
|
||||
stem=stem,
|
||||
title=self.title,
|
||||
upload_date=self.upload_date,
|
||||
url=self.url,
|
||||
validated=self.validated,
|
||||
view_count=self.view_count,
|
||||
video_path=self.video_path.relative_to(get_media_directory()),
|
||||
viewed=self.viewed,
|
||||
)
|
||||
return d
|
||||
|
||||
def validate(self, session: Session):
|
||||
"""Perform a validation of this video and it's files. Mark this video as validated if no errors occur."""
|
||||
if not self.video_file:
|
||||
def validate(self):
|
||||
"""Perform a validation of this video and it's files."""
|
||||
if not self.file_group.primary_path:
|
||||
# Can't validate if there is no video file.
|
||||
return False
|
||||
logger.error(f'Unable to validate video {self.id} without primary file!')
|
||||
|
||||
from .lib import validate_video
|
||||
try:
|
||||
validate_video(self, self.channel.generate_posters if self.channel else False, session)
|
||||
self.validated = True
|
||||
validate_video(self, self.channel.generate_posters if self.channel else False)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to validate video {self}', exc_info=e)
|
||||
|
||||
return self.validated
|
||||
|
||||
def find_my_files(self):
|
||||
"""Search near the video file for it's associated files. (poster, caption, etc.)"""
|
||||
from modules.videos.common import match_video_files
|
||||
if not self.video_path:
|
||||
raise ValueError("Can't find files when I don't have a video path!")
|
||||
|
||||
# Associated files share the same stem, but different extensions.
|
||||
associated_paths = glob_shared_stem(self.video_path)
|
||||
associated_paths.pop(associated_paths.index(self.video_path))
|
||||
|
||||
# Get the session from the video first, fallback to the video file.
|
||||
session: Session = Session.object_session(self) or Session.object_session(self.video_file)
|
||||
|
||||
files = [File.upsert(i, session) for i in associated_paths]
|
||||
_, poster_file, description_file, caption_file, info_json_file = match_video_files(files)
|
||||
|
||||
if poster_file:
|
||||
poster_file.do_index()
|
||||
poster_file.associated = True
|
||||
if caption_file:
|
||||
caption_file.do_index()
|
||||
caption_file.associated = True
|
||||
if info_json_file:
|
||||
info_json_file.do_index()
|
||||
info_json_file.associated = True
|
||||
|
||||
session.flush([i for i in files])
|
||||
|
||||
self.poster_file = poster_file
|
||||
self.caption_file = caption_file
|
||||
self.info_json_file = info_json_file
|
||||
self.file_group.model = Video.__tablename__
|
||||
self.file_group.a_text = self.file_group.title
|
||||
self.file_group.c_text = self.get_video_description()
|
||||
# self.file_group.d_text is handled in `validate_video`.
|
||||
|
||||
@staticmethod
|
||||
def find_by_path(path, session):
|
||||
video = session.query(Video).filter_by(video_path=path).one_or_none()
|
||||
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
|
||||
str_paths = list(map(str, paths))
|
||||
file_group = session.query(FileGroup).filter(FileGroup.primary_path == any_(str_paths)).one_or_none()
|
||||
if not file_group:
|
||||
# This video has not been downloaded previously.
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
|
||||
# Video may have been downloaded previously.
|
||||
video = session.query(Video).filter(Video.file_group_id == file_group.id).one_or_none()
|
||||
if not video:
|
||||
video = Video(file_group=file_group)
|
||||
|
||||
video.validate()
|
||||
session.add(video)
|
||||
session.flush([video, file_group])
|
||||
return video
|
||||
|
||||
@property
|
||||
def info_json_file(self) -> Optional[dict]:
|
||||
for file in self.file_group.my_json_files():
|
||||
return file
|
||||
|
||||
@property
|
||||
def info_json_path(self) -> Optional[pathlib.Path]:
|
||||
if info_json_file := self.info_json_file:
|
||||
return info_json_file['path']
|
||||
|
||||
@property
|
||||
def video_path(self) -> Optional[pathlib.Path]:
|
||||
if self.file_group.primary_path and get_mimetype(self.file_group.primary_path).startswith('video/'):
|
||||
return self.file_group.primary_path
|
||||
|
||||
# No primary file somehow, return the first video file.
|
||||
for file_group in self.file_group.my_video_files():
|
||||
return file_group['path']
|
||||
|
||||
@property
|
||||
def poster_file(self) -> Optional[dict]:
|
||||
for file_group in self.file_group.my_poster_files():
|
||||
return file_group
|
||||
|
||||
@property
|
||||
def poster_path(self) -> Optional[pathlib.Path]:
|
||||
if poster_file := self.poster_file:
|
||||
return poster_file['path']
|
||||
|
||||
@property
|
||||
def caption_files(self) -> List[dict]:
|
||||
return self.file_group.my_files('text/srt') + self.file_group.my_files('text/vtt')
|
||||
|
||||
@property
|
||||
def caption_paths(self) -> List[pathlib.Path]:
|
||||
return [i['path'] for i in self.caption_files]
|
||||
|
||||
def get_caption_text(self) -> Optional[str]:
|
||||
"""Search the FileGroup's files for a caption file. Return the captions from only the best caption file."""
|
||||
caption_paths = self.caption_paths
|
||||
# Some SRT files are more supported than others, these are their preferred order.
|
||||
caption_text = None
|
||||
if english_srt := [i for i in caption_paths if i.name.endswith('.en.srt')]:
|
||||
caption_text = read_captions(english_srt[0])
|
||||
elif srt := [i for i in caption_paths if i.name.endswith('.srt')]:
|
||||
caption_text = read_captions(srt[0])
|
||||
elif english_vtt := [i for i in caption_paths if i.name.endswith('.en.vtt')]:
|
||||
caption_text = read_captions(english_vtt[0])
|
||||
elif vtt := [i for i in caption_paths if i.name.endswith('.vtt')]:
|
||||
caption_text = read_captions(vtt[0])
|
||||
|
||||
return caption_text
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def find_by_path(path, session: Session) -> Optional['Video']:
|
||||
video = session.query(Video) \
|
||||
.join(FileGroup, FileGroup.id == Video.file_group_id) \
|
||||
.filter(FileGroup.primary_path == path).one_or_none()
|
||||
return video
|
||||
|
||||
@staticmethod
|
||||
def find_by_paths(paths: List[pathlib.Path], session) -> List:
|
||||
videos = list(session.query(Video).filter(Video.video_path.in_(paths)))
|
||||
return videos
|
||||
@optional_session
|
||||
def find_by_id(id_: int, session: Session) -> Optional['Video']:
|
||||
video = session.query(Video).filter(Video.id == id_).one_or_none()
|
||||
return video
|
||||
|
||||
@property
|
||||
def primary_path(self):
|
||||
return self.video_path
|
||||
def add_tag(self, tag_or_tag_name: Union[Tag, str]) -> TagFile:
|
||||
tag = Tag.find_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name
|
||||
return self.file_group.add_tag(tag)
|
||||
|
||||
|
||||
class Channel(ModelHelper, Base):
|
||||
|
|
@ -342,7 +304,7 @@ class Channel(ModelHelper, Base):
|
|||
source_id = Column(String)
|
||||
refreshed = Column(Boolean, default=False)
|
||||
|
||||
info_json = Column(JSON)
|
||||
info_json = deferred(Column(JSON))
|
||||
info_date = Column(Date)
|
||||
|
||||
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
|
||||
|
|
@ -414,7 +376,6 @@ class Channel(ModelHelper, Base):
|
|||
calculate_duration=self.calculate_duration,
|
||||
directory=str(self.directory),
|
||||
download_frequency=self.download_frequency,
|
||||
favorites={},
|
||||
generate_posters=self.generate_posters,
|
||||
match_regex=self.match_regex or '',
|
||||
name=self.name,
|
||||
|
|
@ -464,9 +425,10 @@ class Channel(ModelHelper, Base):
|
|||
SELECT
|
||||
SUM(size),
|
||||
MAX(size),
|
||||
COUNT(id)
|
||||
COUNT(video.id)
|
||||
FROM video
|
||||
WHERE channel_id = %(id)s AND video_path IS NOT NULL
|
||||
LEFT JOIN file_group fg on fg.id = video.file_group_id
|
||||
WHERE channel_id = %(id)s
|
||||
'''
|
||||
curs.execute(stmt, dict(id=self.id))
|
||||
size, largest_video, video_count = curs.fetchone()
|
||||
|
|
@ -484,20 +446,26 @@ class Channel(ModelHelper, Base):
|
|||
self_id = self.id
|
||||
|
||||
# Refresh all files within this channel's directory first.
|
||||
await refresh_directory_files_recursively(self.directory, send_events=send_events)
|
||||
await refresh_files(self.directory, send_events=send_events)
|
||||
|
||||
# Apply any info_json (discover censored videos, etc.) second.
|
||||
from modules.videos.common import apply_info_json
|
||||
# Apply any info_json (update view counts) second.
|
||||
from modules.videos.common import update_view_counts
|
||||
if PYTEST:
|
||||
apply_info_json(self_id)
|
||||
update_view_counts(self_id)
|
||||
self.refreshed = True
|
||||
else:
|
||||
|
||||
# Perform info_json in background task. Channel will be marked as refreshed after this completes.
|
||||
async def _():
|
||||
apply_info_json(self_id)
|
||||
update_view_counts(self_id)
|
||||
with get_db_session(commit=True) as session:
|
||||
channel: Channel = session.query(Channel).filter(Channel.id == self_id).one()
|
||||
channel.refreshed = True
|
||||
|
||||
background_task(_())
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional['Channel']:
|
||||
channel = session.query(Channel).filter_by(id=id_).one_or_none()
|
||||
return channel
|
||||
|
|
|
|||
|
|
@ -139,11 +139,12 @@ class VideoResponse:
|
|||
@dataclass
|
||||
class VideoSearchRequest:
|
||||
search_str: Optional[str] = None
|
||||
filters: List[str] = field(default_factory=list)
|
||||
tags: List[str] = field(default_factory=list)
|
||||
offset: Optional[int] = None
|
||||
limit: Optional[int] = VIDEO_QUERY_LIMIT
|
||||
order_by: Optional[str] = DEFAULT_VIDEO_ORDER
|
||||
channel_id: Optional[int] = None
|
||||
tag_names: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -158,15 +159,9 @@ class PaginationQuery:
|
|||
|
||||
|
||||
@dataclass
|
||||
class FavoriteRequest:
|
||||
class TagRequest:
|
||||
video_id: int
|
||||
favorite: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class FavoriteResponse:
|
||||
video_id: int
|
||||
favorite: datetime
|
||||
tag_name: str
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -177,7 +172,6 @@ class WROLModeRequest:
|
|||
@dataclass
|
||||
class VideoStatistics:
|
||||
videos: int
|
||||
favorites: int
|
||||
week: int
|
||||
month: int
|
||||
year: int
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import json
|
||||
import shutil
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -7,101 +6,106 @@ import pytest
|
|||
|
||||
from modules.videos.models import Video, Channel
|
||||
from modules.videos.video import lib as video_lib
|
||||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_videos_index(test_session, test_directory, video_factory):
|
||||
"""A video discovered during refresh has files associated with it. Only the video file is indexed."""
|
||||
"""The video modeler indexes video data into the Video's FileGroup."""
|
||||
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
|
||||
test_session.commit()
|
||||
|
||||
test_session.query(File).delete()
|
||||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
|
||||
for _ in range(1):
|
||||
# Refreshing twice does not change the results.
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(File).count() == 4
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_file.indexed is True, 'Video was not indexed'
|
||||
assert video.video_file.a_text, 'Video title was not indexed'
|
||||
assert video.video_file.d_text, 'Video captions were not indexed'
|
||||
|
||||
assert not video.poster_file.indexed
|
||||
assert not video.caption_file.indexed
|
||||
assert not video.info_json_file.indexed
|
||||
assert video.file_group.indexed is True, 'Video was not indexed'
|
||||
assert video.file_group.a_text and video.file_group.title, 'Video title was not indexed'
|
||||
assert video.file_group.d_text, 'Video captions were not indexed'
|
||||
|
||||
# Delete caption file, the video should be indexed again.
|
||||
video: Video = test_session.query(Video).one()
|
||||
video.caption_path.unlink()
|
||||
for path in video.caption_paths:
|
||||
path.unlink()
|
||||
|
||||
await refresh_files()
|
||||
assert video.video_file.indexed is True, 'Video was not indexed'
|
||||
assert video.video_file.a_text, 'Video title was not indexed'
|
||||
assert not video.video_file.d_text, 'Video captions were not removed'
|
||||
assert video.file_group.indexed is True, 'Video was not indexed'
|
||||
assert video.file_group.a_text and video.file_group.title, 'Video title was not indexed'
|
||||
assert not video.file_group.d_text, 'Video captions were not removed'
|
||||
|
||||
|
||||
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
|
||||
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory, video_file_factory,
|
||||
image_bytes_factory):
|
||||
subdir = test_directory / 'subdir'
|
||||
subdir.mkdir()
|
||||
|
||||
# video1 is in a subdirectory, move its files into the subdirectory.
|
||||
video1 = video_factory(channel_id=simple_channel.id, with_video_file=subdir / 'video1.mp4',
|
||||
with_info_json=True,
|
||||
with_poster_ext='jpg')
|
||||
with_info_json=True, with_poster_ext='jpg')
|
||||
test_session.commit()
|
||||
test_session.delete(video1.poster_file)
|
||||
video1.poster_path = None
|
||||
video1.file_group.size = video1.file_group.data = None
|
||||
video1.file_group.files = list()
|
||||
# video2 is in the test directory.
|
||||
video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
|
||||
with_poster_ext='jpg')
|
||||
video2.poster_path = video2.poster_file = video1.poster_path = None
|
||||
video2.file_group.data = None
|
||||
video2.file_group.files = []
|
||||
test_session.commit()
|
||||
|
||||
assert not video1.size, 'video1 should not have size during creation'
|
||||
assert not video1.file_group.size, 'video1 should not have size during creation'
|
||||
|
||||
# Create a video not in the DB.
|
||||
vid3 = test_directory / 'vid3.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3)
|
||||
vid3 = video_file_factory(test_directory / 'vid3.mp4')
|
||||
|
||||
# Orphan poster should be ignored.
|
||||
orphan_poster = Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg')
|
||||
orphan_poster.touch()
|
||||
orphan_poster.write_bytes(image_bytes_factory())
|
||||
|
||||
# Create a bogus video in the channel.
|
||||
video_file = File(path=test_directory / 'foo')
|
||||
video_file.idempotency = None # File was not added in the current refresh.
|
||||
bogus = Video(video_file=video_file, channel_id=simple_channel.id)
|
||||
test_session.add(bogus)
|
||||
test_session.commit()
|
||||
video_file = test_directory / 'foo.mp4'
|
||||
with get_db_curs(commit=True) as curs:
|
||||
full_stem, _ = split_path_stem_and_suffix(video_file, full=True)
|
||||
stmt = "INSERT INTO file_group (mimetype, full_stem, primary_path, indexed, files, model)" \
|
||||
" values ('video/mp4', %(full_stem)s, %(primary_path)s, true, %(files)s, 'video') RETURNING id"
|
||||
params = {'full_stem': full_stem, 'primary_path': str(video_file), 'files': list()}
|
||||
curs.execute(stmt, params)
|
||||
video4_id = curs.fetchall()[0][0]
|
||||
stmt = "INSERT INTO video (file_group_id) values (%(video_id)s)"
|
||||
curs.execute(stmt, {'video_id': str(video4_id)})
|
||||
|
||||
test_client.post('/api/files/refresh')
|
||||
|
||||
test_session.expire_all()
|
||||
|
||||
# Posters were found during refresh.
|
||||
assert video1.poster_path
|
||||
assert 'subdir' in str(video1.poster_path)
|
||||
assert video2.poster_path
|
||||
assert video1.file_group.my_poster_files()
|
||||
assert 'subdir' in str(video1.file_group.my_poster_files()[0])
|
||||
assert video2.file_group.my_poster_files()
|
||||
# Missing video3 was found
|
||||
video3: Video = test_session.query(Video).filter_by(id=4).one()
|
||||
assert video3.video_path == vid3
|
||||
assert len(set(test_session.query(Video.video_path))) == 3, 'Should have 3 distinct video files'
|
||||
assert len(set(test_session.query(FileGroup.primary_path).filter(FileGroup.model == 'video'))) == 3, \
|
||||
'Should have 3 distinct video files'
|
||||
# Bogus video was removed.
|
||||
assert not any('foo' in str(i.video_path) for i in test_session.query(Video).all())
|
||||
assert not any('foo.mp4' in str(i.video_path) for i in test_session.query(Video).all())
|
||||
# Orphan file was not deleted.
|
||||
assert orphan_poster.is_file(), 'Orphan poster was removed!'
|
||||
|
||||
assert video1.size, 'video1 size was not found'
|
||||
assert video1.file_group.size, 'video1 size was not found'
|
||||
|
||||
# Remove video1's poster, video1 should be updated.
|
||||
video1_video_path = str(video1.video_path)
|
||||
video1.poster_path.unlink()
|
||||
test_client.post('/api/files/refresh')
|
||||
video1 = test_session.query(Video).filter_by(video_path=video1_video_path).one()
|
||||
video1 = Video.find_by_path(video1_video_path, test_session)
|
||||
assert not video1.poster_path
|
||||
|
||||
|
||||
|
|
@ -133,9 +137,9 @@ def test_channels_with_videos(test_session, test_client, test_directory, channel
|
|||
assert test_session.query(Video).count() == 3, 'Did not find correct amount of video files.'
|
||||
assert {i[0] for i in test_session.query(Channel.name)} == {'channel1', 'channel2'}, 'Channels were changed.'
|
||||
|
||||
vid1: Video = test_session.query(Video).filter_by(video_path=vid1_path).one()
|
||||
vid2: Video = test_session.query(Video).filter_by(video_path=vid2_path).one()
|
||||
vid3: Video = test_session.query(Video).filter_by(video_path=vid3_path).one()
|
||||
vid1 = Video.find_by_path(vid1_path)
|
||||
vid2 = Video.find_by_path(vid2_path)
|
||||
vid3 = Video.find_by_path(vid3_path)
|
||||
|
||||
assert vid1.channel == channel1
|
||||
assert vid2.channel == channel2
|
||||
|
|
@ -156,7 +160,7 @@ def test_api_download_channel(test_session, test_client, simple_channel):
|
|||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
|
||||
def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory):
|
||||
def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, assert_video_search):
|
||||
"""Test that videos can be searched and that their order is by their textsearch rank."""
|
||||
# These captions have repeated letters, so they will be higher in the ranking.
|
||||
videos = [
|
||||
|
|
@ -170,58 +174,56 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
|
|||
video_with_search_factory(title=title, d_text=caption)
|
||||
test_session.commit()
|
||||
|
||||
def assert_search(search_str, expected, limit=20, total=None):
|
||||
d = json.dumps({'search_str': search_str, 'limit': limit})
|
||||
_, resp = test_client.post('/api/videos/search', content=d)
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
response_ids = [i['video']['id'] for i in resp.json['files']]
|
||||
assert response_ids == expected
|
||||
assert resp.json['totals']['files'] == (total or len(expected))
|
||||
|
||||
# Repeated runs should return the same result
|
||||
for _ in range(2):
|
||||
# Only videos with a b are returned, ordered by the amount of b's
|
||||
assert_search('b', [1, 2, 3, 4])
|
||||
assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
|
||||
|
||||
# Only two captions have e
|
||||
assert_search('e', [4, 1])
|
||||
assert_video_search(search_str='e', assert_ids=[4, 1])
|
||||
|
||||
# Only 2 contains 2
|
||||
assert_video_search(search_str='2', assert_ids=[2, ])
|
||||
|
||||
# Only two captions have d
|
||||
assert_search('d', [1, 2])
|
||||
assert_video_search(search_str='d', assert_ids=[1, 2])
|
||||
|
||||
# 5 can be gotten by it's title
|
||||
assert_search('5', [5])
|
||||
assert_video_search(search_str='5', assert_ids=[5, ])
|
||||
|
||||
# only video 1 has e and d
|
||||
assert_search('e d', [1])
|
||||
assert_video_search(search_str='e d', assert_ids=[1, ])
|
||||
|
||||
# video 1 and 4 have b and e, but 1 has more
|
||||
assert_search('b e', [1, 4])
|
||||
assert_video_search(search_str='b e', assert_ids=[1, 4])
|
||||
|
||||
# Check totals are correct even with a limit
|
||||
assert_search('b', [1, 2], limit=2, total=4)
|
||||
assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
|
||||
|
||||
|
||||
def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel):
|
||||
def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel, tag_factory):
|
||||
"""Search the Video table. This does not need to use a join with the File table."""
|
||||
vid1 = video_factory(upload_date='2022-09-16', with_video_file=True)
|
||||
vid2 = video_factory(upload_date='2022-09-17', with_video_file=True, channel_id=simple_channel.id)
|
||||
vid3 = video_factory(upload_date='2022-09-18', with_video_file=True)
|
||||
vid1.set_favorite(True)
|
||||
vid1 = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
|
||||
vid2 = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2', channel_id=simple_channel.id)
|
||||
vid3 = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
vid1.add_tag(tag1)
|
||||
vid3.add_tag(tag2)
|
||||
test_session.commit()
|
||||
assert test_session.query(Video).count() == 3
|
||||
assert test_session.query(File).count() == 3
|
||||
assert test_session.query(FileGroup).count() == 3
|
||||
|
||||
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='upload_date')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-upload_date')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-upload_date', limit=2)
|
||||
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-upload_date', limit=2, offset=1)
|
||||
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-upload_date', channel_id=simple_channel.id)
|
||||
assert_video_search(assert_total=1, assert_ids=[vid1.id], filters=['favorite'])
|
||||
assert_video_search(assert_total=1, assert_ids=[vid1.id], filters=['favorite'], limit=1)
|
||||
assert_video_search(assert_total=1, assert_ids=[vid1.id], tag_names=[tag1.name])
|
||||
assert_video_search(assert_total=1, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
|
||||
assert_video_search(assert_total=2, assert_ids=[vid3.id, vid1.id], tag_names=[tag1.name, tag2.name])
|
||||
|
||||
# No results, no total is returned from the DB.
|
||||
assert_video_search(assert_total=0, assert_ids=[], filters=['favorite'], offset=1)
|
||||
assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=1)
|
||||
|
||||
# Check all order_by.
|
||||
for order_by in video_lib.VIDEO_ORDERS:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from wrolpi.dates import now
|
|||
from wrolpi.downloader import Download, DownloadFrequency
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from .. import common
|
||||
from ..common import convert_image, apply_info_json, get_video_duration, generate_video_poster, is_valid_poster
|
||||
from ..common import convert_image, update_view_counts, get_video_duration, generate_video_poster, is_valid_poster
|
||||
from ..lib import save_channels_config, get_channels_config, import_channels_config
|
||||
|
||||
|
||||
|
|
@ -101,22 +101,22 @@ def test_update_view_count(test_session, channel_factory, video_factory):
|
|||
check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None})
|
||||
|
||||
# Channel 1 is updated, the other channels are left alone.
|
||||
apply_info_json(channel1.id)
|
||||
update_view_counts(channel1.id)
|
||||
check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None})
|
||||
|
||||
# Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored.
|
||||
apply_info_json(channel2.id)
|
||||
update_view_counts(channel2.id)
|
||||
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None})
|
||||
|
||||
# All videos are updated.
|
||||
apply_info_json(channel3.id)
|
||||
update_view_counts(channel3.id)
|
||||
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
|
||||
|
||||
# An outdated view count will be overwritten.
|
||||
vid = test_session.query(Video).filter_by(id=1).one()
|
||||
vid.view_count = 8
|
||||
check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14})
|
||||
apply_info_json(channel1.id)
|
||||
update_view_counts(channel1.id)
|
||||
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
|
||||
|
||||
|
||||
|
|
@ -130,128 +130,6 @@ def test_generate_video_poster(video_file):
|
|||
assert poster_path.stat().st_size > 0
|
||||
|
||||
|
||||
def test_update_censored_videos(test_session, video_factory, simple_channel):
|
||||
vid1 = video_factory(channel_id=simple_channel.id)
|
||||
vid2 = video_factory(channel_id=simple_channel.id)
|
||||
vid3 = video_factory(channel_id=simple_channel.id)
|
||||
vid4 = video_factory() # should not be censored because it has no channel.
|
||||
test_session.commit()
|
||||
|
||||
def check_censored(expected):
|
||||
for video_id, censored in expected:
|
||||
video = test_session.query(Video).filter_by(id=video_id).one()
|
||||
assert video.censored == censored
|
||||
|
||||
# Videos are not censored by default.
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)])
|
||||
|
||||
# All videos are in the info_json.
|
||||
simple_channel.info_json = {
|
||||
'entries': [
|
||||
dict(id=vid1.source_id, view_count=0),
|
||||
dict(id=vid2.source_id, view_count=0),
|
||||
dict(id=vid3.source_id, view_count=0),
|
||||
]
|
||||
}
|
||||
test_session.commit()
|
||||
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)])
|
||||
|
||||
simple_channel.info_json = {
|
||||
'entries': [
|
||||
dict(id=vid1.source_id, view_count=0), # vid2 is missing
|
||||
dict(id=vid3.source_id, view_count=0),
|
||||
]
|
||||
}
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, True), (vid3.id, False), (vid4.id, False)])
|
||||
|
||||
simple_channel.info_json = {
|
||||
'entries': [
|
||||
dict(id=vid1.source_id, view_count=0), # vid2 is back, vid3 is missing.
|
||||
dict(id=vid2.source_id, view_count=0),
|
||||
]
|
||||
}
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, True), (vid4.id, False)])
|
||||
|
||||
simple_channel.info_json = {
|
||||
'entries': [] # all videos gone
|
||||
}
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
|
||||
|
||||
# Channels without info json preserve their last censored.
|
||||
simple_channel.info_json = None
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
|
||||
|
||||
|
||||
def test_import_favorites(test_session, test_directory, simple_channel, video_factory, test_channels_config):
|
||||
"""
|
||||
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
|
||||
favorited again.
|
||||
|
||||
A Video's favorited status can only be cleared by calling `Video.delete`.
|
||||
"""
|
||||
video_factory(channel_id=simple_channel.id) # never favorited
|
||||
vid2 = video_factory(channel_id=None) # has no channel
|
||||
vid3 = video_factory(channel_id=simple_channel.id)
|
||||
favorite = vid3.favorite = vid2.favorite = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
test_session.commit()
|
||||
vid2_video_path = vid2.video_path
|
||||
|
||||
favorites = {
|
||||
str(simple_channel.directory.relative_to(test_directory)): {vid3.video_path.name: {'favorite': favorite}},
|
||||
'NO CHANNEL': {vid2.video_path.name: {'favorite': favorite}},
|
||||
}
|
||||
|
||||
# Save config, verify that favorite is set.
|
||||
save_channels_config()
|
||||
config = get_channels_config()
|
||||
assert config.favorites == favorites
|
||||
|
||||
def import_and_verify(favorited_ids: List[int]):
|
||||
with mock.patch('modules.videos.lib.get_channel_source_id', lambda i: 'foo'):
|
||||
import_channels_config()
|
||||
for video in test_session.query(Video).all():
|
||||
if video.id in favorited_ids:
|
||||
assert video.favorite
|
||||
else:
|
||||
assert not video.favorite
|
||||
|
||||
# Clear the favorite (as if the DB was wiped), verify that the favorite is imported and set.
|
||||
vid2.favorite = None
|
||||
import_and_verify([vid2.id, vid3.id])
|
||||
|
||||
# Removing the video does not delete the favorite. If the DB is wiped, we do not want to lose our favorites!
|
||||
test_session.query(Video).filter_by(id=vid2.id).delete()
|
||||
test_session.commit()
|
||||
save_channels_config()
|
||||
config = get_channels_config()
|
||||
assert config.favorites == favorites
|
||||
import_and_verify([vid3.id])
|
||||
|
||||
# Add vid2 again, it should be favorited on import.
|
||||
vid2 = Video(video_path=vid2_video_path)
|
||||
test_session.add(vid2)
|
||||
test_session.commit()
|
||||
import_and_verify([vid3.id, vid2.id])
|
||||
|
||||
# Deleting the Video in the model really removes the favorite status.
|
||||
vid3.delete()
|
||||
config = get_channels_config()
|
||||
assert config.favorites == {'NO CHANNEL': {vid2.video_path.name: {'favorite': favorite}}}
|
||||
import_and_verify([vid2.id])
|
||||
|
||||
|
||||
def test_import_channel_downloads(test_session, channel_factory, test_channels_config):
|
||||
"""Importing the Channels' config should create any missing download records"""
|
||||
channel1 = channel_factory(source_id='foo')
|
||||
|
|
@ -305,67 +183,6 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
|
|||
assert next_download == str(download.next_download)
|
||||
|
||||
|
||||
def test_import_channels_config_outdated(test_session, test_directory, channel_factory, test_channels_config,
|
||||
video_factory):
|
||||
"""The Channels' config used to use a "link" to differentiate channels, test that old configs can be imported."""
|
||||
channel1 = channel_factory(name='Channel1', download_frequency=DownloadFrequency.weekly)
|
||||
channel2 = channel_factory(name='Channel2', download_frequency=DownloadFrequency.weekly)
|
||||
test_session.commit()
|
||||
|
||||
vid1 = video_factory(title='vid1', channel_id=channel1.id)
|
||||
vid2 = video_factory(title='vid2', channel_id=channel2.id)
|
||||
vid3 = video_factory(title='vid3')
|
||||
vid4 = video_factory(title='vid4')
|
||||
test_session.commit()
|
||||
|
||||
channel1_name, channel_1_directory = channel1.name, channel1.directory
|
||||
channel2_name, channel_2_directory = channel2.name, channel2.directory
|
||||
|
||||
assert channel1.directory.exists() and channel1.directory.is_absolute()
|
||||
assert channel2.directory.exists() and channel2.directory.is_absolute()
|
||||
|
||||
# Channel's used to have a relative directory in the DB.
|
||||
test_session.execute('UPDATE channel SET directory=:directory WHERE id=2',
|
||||
dict(directory=str(channel_2_directory.relative_to(test_directory))),
|
||||
)
|
||||
|
||||
save_channels_config()
|
||||
|
||||
# The config has a list of Channels, use the old method of a "dict" with the sanitized name as the key.
|
||||
channels_config = get_channels_config()
|
||||
channels_config.channels = {sanitize_link(i['name']): i for i in channels_config.channels}
|
||||
# Change frequency to verify that the channels are updated.
|
||||
for link, channel in channels_config.channels.items():
|
||||
channel['download_frequency'] = DownloadFrequency.biweekly
|
||||
# Change the favorites to the old "link" method as well.
|
||||
channels_config.favorites = {
|
||||
sanitize_link(channel1.name): {str(vid1.video_path): {'favorite': now()}},
|
||||
sanitize_link(channel2.name): {str(vid2.video_path): {'favorite': now()}},
|
||||
'NO CHANNEL': {str(vid3.video_path): {'favorite': now()}},
|
||||
}
|
||||
channels_config.save()
|
||||
|
||||
assert not vid1.favorite and not vid2.favorite and not vid3.favorite and not vid3.favorite
|
||||
|
||||
assert test_session.query(Channel).count() == 2
|
||||
|
||||
with mock.patch('modules.videos.lib.get_channel_source_id') as mock_get_channel_source_id:
|
||||
mock_get_channel_source_id.return_value = 'some source id'
|
||||
import_channels_config()
|
||||
|
||||
# No new channels were created. Existing channels were updated.
|
||||
assert test_session.query(Channel).count() == 2
|
||||
channel1, channel2 = test_session.query(Channel).order_by(Channel.id)
|
||||
assert channel1.name == channel1_name and str(channel1.directory) == str(channel_1_directory)
|
||||
# Channel2's directory is now absolute.
|
||||
assert channel2.name == channel2_name and str(channel2.directory) == str(channel_2_directory)
|
||||
assert channel1.download_frequency == DownloadFrequency.biweekly
|
||||
assert channel2.download_frequency == DownloadFrequency.biweekly
|
||||
# Videos were favorited by their channel "link".
|
||||
assert vid1.favorite and vid2.favorite and vid3.favorite
|
||||
assert not vid4.favorite
|
||||
|
||||
|
||||
def test_check_for_video_corruption(video_file, test_directory):
|
||||
# The test video is not corrupt.
|
||||
assert common.check_for_video_corruption(video_file) is False
|
||||
|
|
|
|||
|
|
@ -13,27 +13,15 @@ from modules.videos.models import Channel, Video
|
|||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Download, DownloadResult
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.test.common import skip_circleci
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_all_missing_videos(test_session, channel_factory, video_factory):
|
||||
async def test_find_all_missing_videos(test_session, channel_factory, video_factory, image_file):
|
||||
channel1 = channel_factory(url='some url')
|
||||
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]}
|
||||
test_session.commit()
|
||||
|
||||
# Two videos are already downloaded.
|
||||
video_factory()
|
||||
video_factory()
|
||||
|
||||
# No missing videos
|
||||
assert [] == [i async for i in find_all_missing_videos(channel1.id)]
|
||||
|
||||
# Create a video that has no video file.
|
||||
video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo')
|
||||
test_session.add(video)
|
||||
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0, 'webpage_url': 'https://example.com'}]}
|
||||
test_session.commit()
|
||||
|
||||
missing = [i async for i in find_all_missing_videos(channel1.id)]
|
||||
|
|
@ -41,11 +29,25 @@ async def test_find_all_missing_videos(test_session, channel_factory, video_fact
|
|||
|
||||
# The video returned is the one we faked.
|
||||
id_, source_id, entry = missing[0]
|
||||
# Two videos were created for this test already.
|
||||
assert id_ == 3
|
||||
# No Video.id because the video is missing.
|
||||
assert id_ is None
|
||||
# The fake entry we added is regurgitated back.
|
||||
assert entry == channel1.info_json['entries'][0]
|
||||
|
||||
# Create a video without a poster, attempt to download it.
|
||||
video = video_factory(channel_id=channel1.id, with_video_file=True)
|
||||
channel1.info_json = {'entries': [
|
||||
{'id': 'foo', 'view_count': 0, 'webpage_url': 'https://example.com'},
|
||||
{'id': video.source_id, 'view_count': 0, 'webpage_url': 'https://example.com/new'},
|
||||
]}
|
||||
test_session.commit()
|
||||
|
||||
missing = [i async for i in find_all_missing_videos(channel1.id)]
|
||||
assert len(missing) == 2
|
||||
|
||||
_, new = missing
|
||||
assert new == (video.id, video.source_id, channel1.info_json['entries'][1])
|
||||
|
||||
|
||||
example_video_json = {
|
||||
'age_limit': 0,
|
||||
|
|
@ -174,11 +176,12 @@ async def test_download_no_channel(test_session, video_download_manager, video_f
|
|||
assert str(video.video_path) == f'{channel_dir}/video.mp4'
|
||||
|
||||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
|
||||
mock_video_extract_info, mock_video_prepare_filename,
|
||||
mock_video_process_runner):
|
||||
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos.
|
||||
"""Downloading (updating the catalog of) a Channel updates it's info_json.
|
||||
|
||||
If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
|
||||
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
|
||||
|
|
@ -407,10 +410,8 @@ async def test_video_download(test_session, test_directory, simple_channel, vide
|
|||
assert test_session.query(Channel).one()
|
||||
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_path.is_absolute(), 'Video path is not absolute'
|
||||
assert video.poster_path == poster_path, 'Video poster was not discovered'
|
||||
assert video.validated, 'Video was not validated'
|
||||
assert video.video_file and video.video_file.indexed, 'Video was not indexed'
|
||||
assert video.video_path.is_absolute() and video.video_path == video_path, 'Video path is not absolute'
|
||||
assert video.video_path.is_absolute() and video.poster_path == poster_path, 'Video poster was not discovered'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -433,11 +434,10 @@ async def test_download_result(test_session, test_directory, video_download_mana
|
|||
assert download.url == 'https://example.com'
|
||||
assert download.location == '/videos/channel/1/video/1'
|
||||
|
||||
# Video files are associated.
|
||||
# Video has its files.
|
||||
video = test_session.query(Video).one()
|
||||
assert not video.video_file.associated, 'Video file should not be associated.'
|
||||
assert video.video_path.is_file(), 'Video file was not found.'
|
||||
assert video.poster_path.is_file(), 'Poster file was not found.'
|
||||
assert video.poster_file.associated, 'Poster file was not associated.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -527,15 +527,16 @@ example_playlist_json = {
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_playlist(test_session, test_directory, mock_video_extract_info, video_download_manager):
|
||||
async def test_download_playlist(test_session, test_directory, mock_video_extract_info, video_download_manager,
|
||||
video_file):
|
||||
"""All videos in a playlist can be downloaded for it's Channel."""
|
||||
download = Download(url='playlist url')
|
||||
test_session.add(download)
|
||||
channel = get_or_create_channel(example_playlist_json['channel_id'], download.url, example_channel_json['uploader'])
|
||||
video_file = File(path=test_directory / 'video file.mp4')
|
||||
test_session.add(video_file)
|
||||
test_session.add(Video(url='https://www.youtube.com/watch?v=video_1_url',
|
||||
source_id='video 1 id', channel_id=channel.id, video_file=video_file))
|
||||
video = Video.from_paths(test_session, video_file)
|
||||
video.channel_id = channel.id
|
||||
video.source_id = 'video 1 id'
|
||||
video.url = 'https://www.youtube.com/watch?v=video_1_url'
|
||||
test_session.commit()
|
||||
|
||||
mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first.
|
||||
|
|
|
|||
|
|
@ -1,55 +1,19 @@
|
|||
import datetime
|
||||
import pathlib
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
import sqlalchemy
|
||||
|
||||
from modules.videos import lib
|
||||
from modules.videos.common import apply_info_json
|
||||
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
|
||||
from modules.videos.models import Video
|
||||
from modules.videos.video.lib import search_videos
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
def test_search_censored_videos(test_session, simple_channel, video_factory):
|
||||
for i in map(str, range(50)):
|
||||
video_factory(channel_id=simple_channel.id, source_id=i)
|
||||
vid = video_factory(source_id=51) # this should never be modified because it has no channel
|
||||
test_session.add(vid)
|
||||
test_session.commit()
|
||||
|
||||
def set_entries(entries):
|
||||
simple_channel.info_json = {
|
||||
'entries': [{'id': j, 'view_count': 0} for j in entries]} if entries else sqlalchemy.null()
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
test_session.commit()
|
||||
|
||||
# All source_ids are in the entries.
|
||||
set_entries(map(str, range(50)))
|
||||
videos, total = search_videos(filters=['censored'], order_by='id', limit=20)
|
||||
assert [i['video']['source_id'] for i in videos] == []
|
||||
assert total == 0
|
||||
|
||||
# First 5 are censored.
|
||||
set_entries(map(str, range(5, 50)))
|
||||
videos, total = search_videos(filters=['censored'], order_by='id', limit=20)
|
||||
assert [i['video']['source_id'] for i in videos] == [str(i) for i in range(5)]
|
||||
assert all(i['video']['censored'] for i in videos)
|
||||
assert total == 5
|
||||
|
||||
# First 25 are censored.
|
||||
set_entries(map(str, range(25, 50)))
|
||||
videos, total = search_videos(filters=['censored'], order_by='id', limit=20)
|
||||
assert [i['video']['source_id'] for i in videos] == [str(i) for i in range(20)]
|
||||
assert all(i['video']['censored'] for i in videos)
|
||||
assert total == 25
|
||||
|
||||
|
||||
@pytest.mark.parametrize('file_name,expected', [
|
||||
('channel_20000101_12345678910_ some title.mp4', ('channel', '20000101', '12345678910', 'some title')),
|
||||
('channel name_NA_12345678910_ some title.mp4', ('channel name', None, '12345678910', 'some title')),
|
||||
|
|
@ -75,57 +39,6 @@ def test_parse_video_file_name(file_name, expected):
|
|||
assert parse_video_file_name(video_path) == expected, f'{file_name} != {expected}'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_associated_video_files(test_session, test_directory, video_file, image_file, vtt_file1):
|
||||
"""A Video file has files related to it. They should be attached to the Video record."""
|
||||
caption_file = video_file.with_suffix('.en.vtt')
|
||||
vtt_file1.rename(caption_file)
|
||||
poster_file = video_file.with_suffix('.jpeg')
|
||||
image_file.rename(poster_file)
|
||||
|
||||
await files_lib.refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
# All files were found.
|
||||
file_paths = {i.path for i in test_session.query(File)}
|
||||
assert len(file_paths) == 3
|
||||
|
||||
video: Video = test_session.query(Video).one()
|
||||
video_id = video.id
|
||||
assert video.video_file == video_file
|
||||
assert video.poster_file == poster_file
|
||||
assert video.caption_file == caption_file
|
||||
assert video.video_file.modification_datetime
|
||||
assert video.video_file.size
|
||||
assert video.video_file.indexed
|
||||
|
||||
await files_lib.refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
# No new files were added.
|
||||
assert {i.path for i in test_session.query(File)} == file_paths
|
||||
# Same video exists.
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.id == video_id, 'Video id changed. Was the video recreated?'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unassociated_video(test_session, test_directory, video_file):
|
||||
"""A Video file without associated files is still a Video."""
|
||||
# Move the video file into a new subdirectory. The video file should still be found.
|
||||
sub_dir = test_directory / 'subdir'
|
||||
sub_dir.mkdir()
|
||||
new_video_file_path = sub_dir / video_file.name
|
||||
video_file.rename(new_video_file_path)
|
||||
video_file = new_video_file_path
|
||||
|
||||
await files_lib.refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_file == video_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_factory(test_session, video_factory, channel_factory):
|
||||
"""
|
||||
|
|
@ -142,35 +55,42 @@ async def test_video_factory(test_session, video_factory, channel_factory):
|
|||
)
|
||||
test_session.commit()
|
||||
|
||||
assert video.video_path and video.video_file and video.video_path.is_file()
|
||||
assert video.caption_path and video.caption_file and video.caption_path.is_file()
|
||||
assert video.poster_path and video.poster_file and video.poster_path.is_file()
|
||||
assert video.info_json_path and video.info_json_file and video.info_json_path.is_file()
|
||||
assert video.video_path and video.video_path.is_file()
|
||||
assert video.caption_paths and all(i.is_file() for i in video.caption_paths)
|
||||
assert video.poster_path and video.poster_path.is_file()
|
||||
assert video.info_json_path and video.info_json_path.is_file()
|
||||
|
||||
assert video.video_file.a_text # title
|
||||
assert video.video_file.b_text is None
|
||||
assert video.video_file.c_text and 'hello' in video.video_file.c_text # description
|
||||
assert video.video_file.d_text # captions
|
||||
assert video.file_group.a_text # title
|
||||
assert video.file_group.b_text is None
|
||||
assert video.file_group.c_text
|
||||
assert video.file_group.d_text # captions
|
||||
assert video.source_id == 'some id'
|
||||
|
||||
|
||||
def test_validate_video(test_session, test_directory, video_factory):
|
||||
def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file):
|
||||
"""A video poster will be generated only if the channel permits."""
|
||||
vid1 = video_factory(with_video_file=True)
|
||||
video_file = video_file.rename(test_directory / 'Channel Name_20000101_1234567890_The Title.mp4')
|
||||
vid1 = video_factory(with_video_file=video_file)
|
||||
# Clear source_id so source_id in the file will be extracted.
|
||||
vid1.source_id = None
|
||||
assert not vid1.poster_path
|
||||
|
||||
validate_video(vid1, True, test_session)
|
||||
validate_video(vid1, True)
|
||||
assert vid1.video_path == video_file
|
||||
assert vid1.poster_path, 'Poster was not created'
|
||||
assert vid1.poster_path.is_file(), 'Poster path does not exist'
|
||||
assert vid1.video_path.stem == vid1.poster_path.stem
|
||||
assert vid1.poster_path.suffix == '.jpg'
|
||||
assert vid1.upload_date == datetime.datetime(2000, 1, 1, 0, 1, tzinfo=datetime.timezone.utc)
|
||||
assert vid1.source_id == '1234567890'
|
||||
assert vid1.file_group.title == 'The Title'
|
||||
|
||||
# A PNG is replaced.
|
||||
vid2 = video_factory(with_video_file=True, with_poster_ext='.png')
|
||||
vid2.poster_path.with_suffix('.jpg').touch() # New poster exists, replace it.
|
||||
vid2.poster_path.with_suffix('.jpg').write_bytes(image_bytes_factory()) # New poster exists, replace it.
|
||||
assert vid2.poster_path and vid2.poster_path.is_file() and vid2.poster_path.suffix == '.png', \
|
||||
'Poster was not initialized'
|
||||
validate_video(vid2, True, test_session)
|
||||
validate_video(vid2, True)
|
||||
assert vid2.poster_path.is_file(), 'New poster was not generated'
|
||||
assert vid2.poster_path.suffix == '.jpg'
|
||||
|
||||
|
|
@ -197,34 +117,62 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
|
|||
@pytest.mark.asyncio
|
||||
async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory):
|
||||
# A Video without associated files is not orphaned.
|
||||
vid1 = video_factory(with_video_file=True)
|
||||
vid1 = video_factory(title='vid1', with_video_file=True)
|
||||
# The video files will be removed...
|
||||
vid2 = video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpeg', with_info_json=True)
|
||||
vid3 = video_factory(with_video_file=True, with_poster_ext='jpg')
|
||||
vid2 = video_factory(title='vid2', with_video_file=True, with_caption_file=True, with_poster_ext='jpeg',
|
||||
with_info_json=True)
|
||||
vid3 = video_factory(title='vid3', with_video_file=True, with_poster_ext='jpg')
|
||||
# This will be ignored because it is not in the "videos" subdirectory.
|
||||
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', test_directory / 'video4.en.vtt')
|
||||
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', test_directory / 'vid4.en.vtt')
|
||||
test_session.commit()
|
||||
|
||||
# Remove vid2 video. Caption, poster, info_json are now orphaned.
|
||||
vid2.video_path.unlink()
|
||||
vid2_caption_path, vid2_poster_path, vid2_info_json_path = vid2.caption_path, vid2.poster_path, vid2.info_json_path
|
||||
vid2_caption_paths, vid2_poster_path, vid2_info_json_path = vid2.caption_paths, vid2.poster_path, vid2.info_json_path
|
||||
# Remove vid3 video. Poster is now orphaned.
|
||||
vid3.video_path.unlink()
|
||||
vid3_poster_path = vid3.poster_path
|
||||
await refresh_files()
|
||||
await files_lib.refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
# 6 files when two video files are deleted.
|
||||
# (vid1, vid2[caption,poster,json], vid3 poster, vid4)
|
||||
assert test_session.query(File).count() == 6
|
||||
assert test_session.query(FileGroup).count() == 4
|
||||
assert test_session.query(Video).count() == 1
|
||||
|
||||
videos_directory = test_directory / 'videos'
|
||||
results = lib.find_orphaned_video_files(videos_directory)
|
||||
|
||||
assert sorted(list(results)) == sorted([
|
||||
vid2_caption_path,
|
||||
vid2_caption_paths[0],
|
||||
vid2_info_json_path,
|
||||
vid2_poster_path,
|
||||
vid3_poster_path,
|
||||
])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_channels_config_favorites_tags(test_session, test_directory, test_channels_config, video_factory,
|
||||
simple_channel):
|
||||
"""Video favorites have been moved to Tags. Any Favorited video should be tagged with Favorite tag."""
|
||||
test_session.add(Tag(name='Favorite'))
|
||||
video1 = video_factory(channel_id=simple_channel.id)
|
||||
video2 = video_factory(channel_id=simple_channel.id)
|
||||
test_session.commit()
|
||||
|
||||
# Favorite video1 in config.
|
||||
channels_config = lib.get_channels_config()
|
||||
channels_config.favorites = {
|
||||
str(simple_channel.directory.relative_to(test_directory)): {
|
||||
str(video1.video_path.relative_to(simple_channel.directory)): now().isoformat(),
|
||||
}
|
||||
}
|
||||
|
||||
# The import function will tag the Video with Favorite.
|
||||
lib.import_channels_config()
|
||||
|
||||
# video1 has Favorite tag. It is also in the config.
|
||||
assert video1.file_group.tag_files and video1.file_group.tag_files[0].tag.name == 'Favorite'
|
||||
assert str(video1.file_group.primary_path.relative_to(test_directory)) in test_channels_config.read_text()
|
||||
# video2 was not tagged.
|
||||
assert not video2.file_group.tag_files
|
||||
|
|
|
|||
|
|
@ -1,45 +1,73 @@
|
|||
import pytest
|
||||
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files.models import File
|
||||
|
||||
|
||||
def test_timezone(test_session):
|
||||
"""
|
||||
A Video's dates should have a TimeZone.
|
||||
"""
|
||||
video = Video(upload_date=now())
|
||||
test_session.add(video)
|
||||
test_session.commit()
|
||||
assert video.upload_date.tzinfo
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
def test_delete_video_no_channel(test_session, simple_video):
|
||||
"""
|
||||
A Video can be deleted even if it does not have a channel.
|
||||
"""
|
||||
"""A Video can be deleted even if it does not have a channel."""
|
||||
assert simple_video.channel
|
||||
video_id = simple_video.id
|
||||
simple_video.channel_id = None
|
||||
test_session.commit()
|
||||
|
||||
simple_video: Video = test_session.query(Video).filter_by(id=video_id).one()
|
||||
paths = list(simple_video.file_group.my_paths())
|
||||
|
||||
assert not simple_video.channel
|
||||
simple_video.delete()
|
||||
|
||||
# All files were deleted.
|
||||
for path in paths:
|
||||
assert not path.exists()
|
||||
|
||||
def test_delete_video(test_session, simple_video, image_file):
|
||||
"""When a Video record is deleted, all referenced file records should be deleted."""
|
||||
simple_video.poster_file = File(path=image_file)
|
||||
test_session.add(simple_video.poster_file)
|
||||
test_session.commit()
|
||||
|
||||
assert simple_video.video_path.is_file(), 'Video file was not created.'
|
||||
assert simple_video.poster_path.is_file(), 'Video poster was not created.'
|
||||
assert test_session.query(Video).count() == 1, 'Video was not created.'
|
||||
assert test_session.query(File).count() == 2, 'Video file and poster file were not created.'
|
||||
assert test_session.query(FileGroup).count() == 0
|
||||
assert test_session.query(Video).count() == 0
|
||||
|
||||
simple_video.delete()
|
||||
|
||||
def test_delete_video(test_session, video_factory):
|
||||
"""When a Video record is deleted, all referenced file records should be deleted."""
|
||||
video = video_factory(with_video_file=True, with_poster_ext='png')
|
||||
test_session.commit()
|
||||
|
||||
assert video.video_path.is_file(), 'Video file was not created.'
|
||||
assert video.poster_path.is_file(), 'Video poster was not created.'
|
||||
assert test_session.query(Video).count() == 1, 'Video was not created.'
|
||||
assert test_session.query(FileGroup).count() == 1, 'Video file and poster file were not created.'
|
||||
|
||||
video.delete()
|
||||
test_session.commit()
|
||||
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
|
||||
assert test_session.query(File).count() == 0, 'Video files were not deleted.'
|
||||
assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_channel_refresh(test_session, test_directory, channel_factory, video_factory):
|
||||
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
|
||||
# Create a video file in this channel's directory.
|
||||
channel = channel_factory()
|
||||
video1_path = channel.directory / 'video1.mp4'
|
||||
video_factory(channel_id=channel.id, with_video_file=video1_path, with_poster_ext='jpg')
|
||||
# Create another video outside the channel's directory. It should not have a channel.
|
||||
video2_path = test_directory / 'video2.mp4'
|
||||
video_factory(with_video_file=video2_path)
|
||||
# Delete all FileGroups so the channel will be associated again.
|
||||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
|
||||
await files_lib.refresh_files()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 2
|
||||
assert test_session.query(Video).count() == 2
|
||||
|
||||
# The video is associated with `simple_channel` because its file is in the Channel's directory.
|
||||
video1: Video = Video.find_by_path(video1_path, test_session)
|
||||
video2: Video = Video.find_by_path(video2_path, test_session)
|
||||
assert video1.channel == channel
|
||||
assert not video2.channel
|
||||
assert video1.__json__()['video']['channel']
|
||||
video_channel = video1.__json__()['video']['channel']
|
||||
assert video_channel['id'] == channel.id
|
||||
assert video_channel['name'] == channel.name
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ logger = logger.getChild(__name__)
|
|||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
def video_get(_: Request, video_id: int):
|
||||
video, previous_video, next_video = lib.get_video_for_app(video_id)
|
||||
return json_response({'file': video, 'prev': previous_video, 'next': next_video})
|
||||
return json_response({'file_group': video, 'prev': previous_video, 'next': next_video})
|
||||
|
||||
|
||||
@video_bp.post('/search')
|
||||
|
|
@ -38,16 +38,16 @@ async def search(_: Request, body: schema.VideoSearchRequest):
|
|||
if body.order_by not in lib.VIDEO_ORDERS:
|
||||
raise InvalidOrderBy('Invalid order by')
|
||||
|
||||
files, videos_total = lib.search_videos(
|
||||
file_groups, videos_total = lib.search_videos(
|
||||
body.search_str,
|
||||
body.offset,
|
||||
body.limit,
|
||||
body.channel_id,
|
||||
body.order_by,
|
||||
body.filters,
|
||||
body.tag_names,
|
||||
)
|
||||
|
||||
ret = {'files': list(files), 'totals': {'files': videos_total}}
|
||||
ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,14 @@
|
|||
from datetime import datetime
|
||||
from typing import Tuple, Optional, List
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.common import run_after, logger
|
||||
from wrolpi.db import get_db_session, optional_session
|
||||
from wrolpi.errors import UnknownVideo
|
||||
from wrolpi.files.lib import handle_search_results
|
||||
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Video
|
||||
|
||||
logger.getChild(__name__)
|
||||
|
||||
|
|
@ -31,47 +30,44 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
|
|||
video.set_viewed()
|
||||
previous_video, next_video = video.get_surrounding_videos()
|
||||
|
||||
caption = video.video_file.d_text
|
||||
video = video.video_file.__json__()
|
||||
video['video']['caption'] = caption
|
||||
previous_video = previous_video.video_file.__json__() if previous_video and previous_video.video_file else None
|
||||
next_video = next_video.video_file.__json__() if next_video and next_video.video_file else None
|
||||
video = video.__json__()
|
||||
previous_video = previous_video.__json__() if previous_video and previous_video.file_group else None
|
||||
next_video = next_video.__json__() if next_video and next_video.file_group else None
|
||||
|
||||
return video, previous_video, next_video
|
||||
|
||||
|
||||
VIDEO_ORDERS = {
|
||||
'upload_date': 'v.upload_date ASC, LOWER(v.video_path) ASC',
|
||||
'-upload_date': 'v.upload_date DESC NULLS LAST, LOWER(v.video_path) DESC',
|
||||
'rank': '2 DESC, LOWER(v.video_path) DESC',
|
||||
'-rank': '2 ASC, LOWER(v.video_path) ASC',
|
||||
'id': 'v.id ASC',
|
||||
'-id': 'v.id DESC',
|
||||
'size': 'f.size ASC, LOWER(v.video_path) ASC',
|
||||
'-size': 'f.size DESC, LOWER(v.video_path) DESC',
|
||||
'duration': 'duration ASC, LOWER(v.video_path) ASC',
|
||||
'-duration': 'duration DESC, LOWER(v.video_path) DESC',
|
||||
'favorite': 'favorite ASC, LOWER(v.video_path) ASC',
|
||||
'-favorite': 'favorite DESC, LOWER(v.video_path) DESC',
|
||||
'upload_date': 'v.upload_date ASC, LOWER(fg.primary_path) ASC',
|
||||
'-upload_date': 'v.upload_date DESC NULLS LAST, LOWER(fg.primary_path) DESC',
|
||||
'rank': '2 DESC, LOWER(fg.primary_path) DESC',
|
||||
'-rank': '2 ASC, LOWER(fg.primary_path) ASC',
|
||||
'id': 'fg.id ASC',
|
||||
'-id': 'fg.id DESC',
|
||||
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
|
||||
'-size': 'fg.size DESC, LOWER(fg.primary_path) DESC',
|
||||
'duration': 'duration ASC, LOWER(fg.primary_path) ASC',
|
||||
'-duration': 'duration DESC, LOWER(fg.primary_path) DESC',
|
||||
'viewed': 'v.viewed ASC',
|
||||
'-viewed': 'v.viewed DESC',
|
||||
'view_count': 'v.view_count ASC',
|
||||
'-view_count': 'v.view_count DESC',
|
||||
'modification_datetime': 'f.modification_datetime ASC',
|
||||
'-modification_datetime': 'f.modification_datetime DESC',
|
||||
'modification_datetime': 'fg.modification_datetime ASC',
|
||||
'-modification_datetime': 'fg.modification_datetime DESC',
|
||||
}
|
||||
NO_NULL_ORDERS = {
|
||||
'viewed': 'v.viewed IS NOT NULL',
|
||||
'-viewed': 'v.viewed IS NOT NULL',
|
||||
'duration': 'v.duration IS NOT NULL',
|
||||
'-duration': 'v.duration IS NOT NULL',
|
||||
'size': 'f.size IS NOT NULL',
|
||||
'-size': 'f.size IS NOT NULL',
|
||||
'size': 'fg.size IS NOT NULL',
|
||||
'-size': 'fg.size IS NOT NULL',
|
||||
'view_count': 'v.view_count IS NOT NULL',
|
||||
'-view_count': 'v.view_count IS NOT NULL',
|
||||
'modification_datetime': 'f.modification_datetime IS NOT NULL',
|
||||
'-modification_datetime': 'f.modification_datetime IS NOT NULL',
|
||||
'modification_datetime': 'fg.modification_datetime IS NOT NULL',
|
||||
'-modification_datetime': 'fg.modification_datetime IS NOT NULL',
|
||||
}
|
||||
JOIN_ORDERS = ('upload_date', '-upload_date', 'viewed', '-viewed', 'view_count', '-view_count', 'duration', '-duration')
|
||||
DEFAULT_VIDEO_ORDER = 'rank'
|
||||
VIDEO_QUERY_LIMIT = 24
|
||||
|
||||
|
|
@ -81,77 +77,77 @@ def search_videos(
|
|||
offset: int = None,
|
||||
limit: int = VIDEO_QUERY_LIMIT,
|
||||
channel_id: int = None,
|
||||
order_by: str = None,
|
||||
filters: List[str] = None,
|
||||
order: str = None,
|
||||
tag_names: List[str] = None,
|
||||
) -> Tuple[List[dict], int]:
|
||||
filters = filters or []
|
||||
wheres = ['v.video_path IS NOT NULL']
|
||||
tag_names = tag_names or []
|
||||
# Only search videos.
|
||||
wheres = ["fg.model = 'video'"]
|
||||
joins = list()
|
||||
join_video = False
|
||||
|
||||
params = dict(search_str=search_str, offset=offset)
|
||||
if channel_id:
|
||||
wheres.append('v.channel_id = %(channel_id)s')
|
||||
joins.append('LEFT JOIN channel c ON c.id = v.channel_id')
|
||||
join_video = True
|
||||
params['channel_id'] = channel_id
|
||||
|
||||
# Apply filters.
|
||||
if 'favorite' in filters:
|
||||
wheres.append('v.favorite IS NOT NULL')
|
||||
if 'censored' in filters:
|
||||
wheres.append('v.censored = true')
|
||||
|
||||
if search_str:
|
||||
# A search_str was provided by the user, modify the query to filter by it.
|
||||
select_columns = 'f.path, ts_rank(f.textsearch, websearch_to_tsquery(%(search_str)s)), ' \
|
||||
'COUNT(*) OVER() AS total'
|
||||
wheres.append('f.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
select_columns = 'fg.id, ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
|
||||
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
params['search_str'] = search_str
|
||||
join = 'LEFT JOIN file f on f.path = v.video_path'
|
||||
else:
|
||||
# No search_str provided. Get path and total only.
|
||||
select_columns = 'v.video_path AS path, COUNT(*) OVER() AS total'
|
||||
join = ''
|
||||
# No search_str provided. Get id and total only.
|
||||
select_columns = 'fg.id, COUNT(*) OVER() AS total'
|
||||
|
||||
if order_by in {'size', '-size', 'modification_datetime', '-modification_datetime'}:
|
||||
# Size and modification_datetime are from the video file.
|
||||
join = 'LEFT JOIN file f on f.path = v.video_path'
|
||||
if tag_names:
|
||||
where_, params_, join_ = tag_names_to_clauses(tag_names)
|
||||
wheres.append(where_)
|
||||
params.update(params_)
|
||||
joins.append(join_)
|
||||
|
||||
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
|
||||
# whitelist.
|
||||
order = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
|
||||
if order_by:
|
||||
order_by = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
|
||||
if order:
|
||||
try:
|
||||
order = VIDEO_ORDERS[order_by]
|
||||
order_by = VIDEO_ORDERS[order]
|
||||
except KeyError:
|
||||
raise
|
||||
if order_by in NO_NULL_ORDERS:
|
||||
wheres.append(NO_NULL_ORDERS[order_by])
|
||||
if order in NO_NULL_ORDERS:
|
||||
wheres.append(NO_NULL_ORDERS[order])
|
||||
if order in JOIN_ORDERS:
|
||||
join_video = True
|
||||
|
||||
if join_video:
|
||||
joins.insert(0, 'LEFT JOIN video v on v.file_group_id = fg.id')
|
||||
|
||||
wheres = '\n AND '.join(wheres)
|
||||
where = f'WHERE\n{wheres}' if wheres else ''
|
||||
join = '\n'.join(joins)
|
||||
stmt = f'''
|
||||
SELECT
|
||||
{select_columns}
|
||||
FROM video v
|
||||
FROM file_group fg
|
||||
{join}
|
||||
{where}
|
||||
ORDER BY {order}
|
||||
ORDER BY {order_by}
|
||||
OFFSET %(offset)s LIMIT {int(limit)}
|
||||
'''.strip()
|
||||
logger.debug(stmt, params)
|
||||
|
||||
results, total = handle_search_results(stmt, params)
|
||||
results, total = handle_file_group_search_results(stmt, params)
|
||||
return results, total
|
||||
|
||||
|
||||
@run_after(save_channels_config)
|
||||
def set_video_favorite(video_id: int, favorite: bool) -> Optional[datetime]:
|
||||
"""
|
||||
Set the Video.favorite to the current datetime if `favorite` is True, otherwise None.
|
||||
"""
|
||||
with get_db_session(commit=True) as session:
|
||||
video = session.query(Video).filter_by(id=video_id).one()
|
||||
favorite = video.set_favorite(favorite)
|
||||
|
||||
return favorite
|
||||
def tag_video(video_id: int, tag_name: str):
|
||||
"""Tag a video"""
|
||||
with get_db_session(commit=True):
|
||||
video = Video.find_by_id(video_id)
|
||||
video.add_tag(tag_name)
|
||||
|
||||
|
||||
@optional_session
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
from json import dumps
|
||||
|
|
@ -60,19 +61,19 @@ def test_get_video_prev_next(test_session, channel_factory, video_factory):
|
|||
if prev_title is None:
|
||||
assert prev_video is None
|
||||
else:
|
||||
assert prev_video and prev_video.title == prev_title
|
||||
assert prev_video and prev_video.file_group.title == prev_title
|
||||
|
||||
if next_title is None:
|
||||
assert next_video is None
|
||||
else:
|
||||
assert next_video and next_video.title == next_title
|
||||
assert next_video and next_video.file_group.title == next_title
|
||||
except AssertionError as e:
|
||||
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
|
||||
|
||||
|
||||
def test_get_video_for_app(test_session, simple_channel, simple_video):
|
||||
vid, prev, next_ = get_video_for_app(simple_video.id)
|
||||
assert vid['video']['id'] == simple_video.id
|
||||
assert vid['id'] == simple_video.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -85,7 +86,7 @@ def test_video_delete(test_client, test_session, test_directory, channel_factory
|
|||
|
||||
assert test_session.query(Video).count() == 2
|
||||
|
||||
vid1_video_path, vid1_caption_path = vid1.video_path, vid1.caption_path
|
||||
vid1_video_path, vid1_caption_path = vid1.video_path, vid1.caption_paths[0]
|
||||
vid2_video_path, vid2_info_json_path = vid2.video_path, vid2.info_json_path
|
||||
|
||||
# No videos have been deleted yet.
|
||||
|
|
@ -120,10 +121,10 @@ def test_video_delete(test_client, test_session, test_directory, channel_factory
|
|||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixture, test_download_manager):
|
||||
def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixture, test_download_manager, tag_factory):
|
||||
"""Many methods are blocked when WROL Mode is enabled."""
|
||||
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
|
||||
favorite = dumps(dict(video_id=simple_video.id, favorite=True))
|
||||
tag = tag_factory()
|
||||
|
||||
wrol_mode_fixture(True)
|
||||
|
||||
|
|
@ -152,8 +153,9 @@ def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixtu
|
|||
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
|
||||
# THE REST OF THESE METHODS ARE ALLOWED
|
||||
_, resp = api_app.test_client.post('/api/videos/favorite', content=favorite)
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
content = dict(video_id=simple_video.id, tag_name=tag.name)
|
||||
_, resp = api_app.test_client.post('/api/videos/tag', content=json.dumps(content))
|
||||
assert resp.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert test_download_manager.stopped.is_set()
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,7 @@ formatters = {
|
|||
|
||||
|
||||
def video_url_resolver(domain: str, entry: dict) -> str:
|
||||
"""
|
||||
Guess the full URL of a Youtube-DL entry.
|
||||
"""
|
||||
"""Guess the full URL of a Youtube-DL entry."""
|
||||
if 'webpage_url' in entry:
|
||||
return entry['webpage_url']
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ Pillow==9.3.0
|
|||
Pint==0.18
|
||||
PyPDF2[crypto]==2.11.2
|
||||
PyYAML==6.0
|
||||
SQLAlchemy==1.3.22
|
||||
SQLAlchemy[asyncio]==1.3.22
|
||||
aiohttp==3.8.3
|
||||
alembic==1.7.5
|
||||
beautifulsoup4==4.10.0
|
||||
|
|
|
|||
|
|
@ -36,14 +36,40 @@ from wrolpi.vars import PYTEST, MODULES_DIR, \
|
|||
|
||||
logger = logging.getLogger()
|
||||
ch = logging.StreamHandler()
|
||||
formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s')
|
||||
formatter = logging.Formatter('[%(asctime)s] [%(name)s:%(lineno)d] [%(levelname)s] %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
logger_ = logger.getChild(__name__)
|
||||
|
||||
|
||||
def set_log_level(level, warn_level: bool = True):
|
||||
"""Set the level of the root logger so all children that have been created (or will be created) share the same
|
||||
level."""
|
||||
logger.setLevel(level)
|
||||
if level == logging.NOTSET:
|
||||
# Custom maximum logging level which will include SQLAlchemy debugging.
|
||||
sa_logger = logging.getLogger('sqlalchemy.engine')
|
||||
sa_logger.setLevel(logging.DEBUG)
|
||||
|
||||
# Always warn about the log level, so we know what should have been logged.
|
||||
effective_level = logger.getEffectiveLevel()
|
||||
level_name = logging.getLevelName(effective_level)
|
||||
if warn_level:
|
||||
logger.warning(f'Logging level: {level_name}')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def log_level_context(level):
|
||||
starting_level = logger.getEffectiveLevel()
|
||||
set_log_level(level, warn_level=False)
|
||||
yield
|
||||
set_log_level(starting_level, warn_level=False)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'logger',
|
||||
'set_log_level',
|
||||
'Base',
|
||||
'ModelHelper',
|
||||
'get_model_by_table_name',
|
||||
|
|
@ -95,7 +121,6 @@ __all__ = [
|
|||
'truncate_object_bytes',
|
||||
'background_task',
|
||||
'get_warn_once',
|
||||
'ordered_unique_list',
|
||||
'truncate_generator_bytes',
|
||||
'cancel_refresh_tasks',
|
||||
'cancelable_wrapper',
|
||||
|
|
@ -123,8 +148,8 @@ class ModelHelper:
|
|||
def find_by_paths(paths, session) -> List:
|
||||
raise NotImplementedError('This model has not defined this method.')
|
||||
|
||||
@property
|
||||
def primary_path(self):
|
||||
@staticmethod
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
|
||||
raise NotImplementedError('This model has not defined this method.')
|
||||
|
||||
|
||||
|
|
@ -622,7 +647,7 @@ def api_param_limiter(maximum: int, default: int = 20) -> callable:
|
|||
>>> limiter(100)
|
||||
100
|
||||
>>> limiter(150)
|
||||
150
|
||||
100
|
||||
"""
|
||||
|
||||
def limiter_(i: int) -> int:
|
||||
|
|
@ -732,12 +757,14 @@ def walk(path: Path) -> Generator[Path, None, None]:
|
|||
yield from walk(path)
|
||||
|
||||
|
||||
def get_files_and_directories(path: Path):
|
||||
def get_files_and_directories(directory: Path):
|
||||
"""Walk a directory, return a tuple of all files and all directories within. (Not recursive)."""
|
||||
if not path.is_dir():
|
||||
if not directory.exists():
|
||||
raise ValueError(f'Directory does not exist: {directory}')
|
||||
if not directory.is_dir():
|
||||
raise ValueError('Can only walk a directory.')
|
||||
|
||||
directories, files = partition(lambda i: i.is_dir(), path.iterdir())
|
||||
directories, files = partition(lambda i: i.is_dir(), directory.iterdir())
|
||||
return files, directories
|
||||
|
||||
|
||||
|
|
@ -798,27 +825,11 @@ def register_modeler(modeler: callable):
|
|||
return modeler
|
||||
|
||||
|
||||
def apply_modelers(files, session: Session):
|
||||
"""
|
||||
Group Files by stem, send those groups to the modelers. Each modeler will model what Files are valid.
|
||||
"""
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
|
||||
if not files:
|
||||
return
|
||||
|
||||
# Group all files by their common name (without the suffix).
|
||||
groups = {}
|
||||
for file in files:
|
||||
file.do_stats()
|
||||
stem, _ = split_path_stem_and_suffix(file.path)
|
||||
try:
|
||||
groups[stem].append(file)
|
||||
except KeyError:
|
||||
groups[stem] = [file, ]
|
||||
|
||||
async def apply_modelers():
|
||||
for modeler in modelers:
|
||||
modeler(groups, session)
|
||||
await modeler()
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
after_refresh = []
|
||||
|
|
@ -829,7 +840,7 @@ def register_after_refresh(func: callable):
|
|||
return func
|
||||
|
||||
|
||||
def apply_after_refresh():
|
||||
async def apply_after_refresh():
|
||||
for func in after_refresh:
|
||||
logger_.info(f'Applying after-refresh {func.__name__}')
|
||||
func()
|
||||
|
|
@ -871,7 +882,7 @@ def chunks_by_stem(it: List[Union[pathlib.Path, str]], size: int) -> Generator[L
|
|||
raise ValueError('size must be a positive integer')
|
||||
|
||||
if not it or len(it) < size:
|
||||
yield it
|
||||
yield sorted(it)
|
||||
return
|
||||
|
||||
if not isinstance(it, list):
|
||||
|
|
@ -1042,11 +1053,6 @@ def get_warn_once(message: str, logger__: logging.Logger, level=logging.ERROR):
|
|||
return warn_once
|
||||
|
||||
|
||||
def ordered_unique_list(lst: Iterable) -> List:
|
||||
"""Return a new list that contains only the first occurrence of each item."""
|
||||
return list(dict.fromkeys(lst))
|
||||
|
||||
|
||||
def get_global_statistics():
|
||||
from wrolpi.db import get_db_curs
|
||||
with get_db_curs() as curs:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,14 @@
|
|||
Fixtures for Pytest tests.
|
||||
"""
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
from itertools import zip_longest
|
||||
from typing import List, Callable, Union, Dict
|
||||
from typing import Tuple, Set
|
||||
from unittest import mock
|
||||
|
|
@ -13,18 +17,21 @@ from unittest.mock import MagicMock
|
|||
from uuid import uuid1, uuid4
|
||||
|
||||
import pytest
|
||||
import sqlalchemy
|
||||
from PIL import Image
|
||||
from sanic_testing.testing import SanicTestClient
|
||||
from sqlalchemy.engine import Engine, create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import iterify
|
||||
from wrolpi.common import iterify, log_level_context
|
||||
from wrolpi.common import set_test_media_directory, Base, set_test_config
|
||||
from wrolpi.dates import set_test_now
|
||||
from wrolpi.db import postgres_engine, get_db_args
|
||||
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.root_api import BLUEPRINTS, api_app
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
|
|
@ -75,6 +82,13 @@ def test_session() -> Session:
|
|||
conn.close()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def test_debug_logger(level: int = logging.DEBUG):
|
||||
"""Tests use debug logging by default."""
|
||||
with log_level_context(level):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def test_directory() -> pathlib.Path:
|
||||
"""
|
||||
|
|
@ -180,6 +194,19 @@ def video_file(test_directory) -> pathlib.Path:
|
|||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_file_factory(test_directory):
|
||||
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
|
||||
|
||||
def _(path: pathlib.Path = None) -> pathlib.Path:
|
||||
destination = path or test_directory / f'{uuid4()}.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
|
||||
|
||||
return destination
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def corrupted_video_file(test_directory) -> pathlib.Path:
|
||||
"""Return a copy of the corrupted video file in the `test_directory`."""
|
||||
|
|
@ -334,3 +361,171 @@ def flags_lock():
|
|||
finally:
|
||||
flags.TESTING_LOCK.clear()
|
||||
FLAGS_LOCK.release()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def tag_factory(test_session):
|
||||
names = ['one', 'two', 'three', 'four', 'five', 'six']
|
||||
count = 1
|
||||
|
||||
def factory() -> Tag:
|
||||
tag = Tag(name=names.pop(0), color=f'#{str(count) * 6}')
|
||||
test_session.add(tag)
|
||||
test_session.commit()
|
||||
return tag
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_files(test_session, test_directory):
|
||||
def _(files: List[Dict]):
|
||||
from wrolpi.files.models import FileGroup
|
||||
for expected in files:
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
full_stem, _ = split_path_stem_and_suffix(test_directory / expected.pop('path'), full=True)
|
||||
try:
|
||||
file: FileGroup = test_session.query(FileGroup).filter_by(full_stem=full_stem).one()
|
||||
except sqlalchemy.orm.exc.NoResultFound as e:
|
||||
raise ValueError(f'No FileGroup found with {full_stem=}') from e
|
||||
|
||||
for key, value in expected.items():
|
||||
attr = getattr(file, key)
|
||||
if attr != value:
|
||||
raise AssertionError(f'{full_stem} {attr} != {value}')
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_file_groups(test_session, test_directory):
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
def _(file_groups: List[Dict], assert_count: bool = True):
|
||||
from wrolpi.files.models import FileGroup
|
||||
for expected in file_groups:
|
||||
if 'full_stem' in expected:
|
||||
full_stem = str(test_directory / expected.pop('full_stem'))
|
||||
try:
|
||||
file_group: FileGroup = test_session.query(FileGroup).filter_by(full_stem=full_stem).one()
|
||||
except sqlalchemy.exc.NoResultFound as e:
|
||||
raise ValueError(f'No FileGroup found with {full_stem=}') from e
|
||||
elif 'primary_path' in expected:
|
||||
primary_path = str(test_directory / expected.pop('primary_path'))
|
||||
try:
|
||||
file_group: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
|
||||
except sqlalchemy.exc.NoResultFound as e:
|
||||
raise ValueError(f'No FileGroup found with {primary_path=}') from e
|
||||
|
||||
# Compare file dictionaries.
|
||||
files = expected.pop('files', None)
|
||||
if files:
|
||||
for exp, actual in zip_longest(files, file_group.files):
|
||||
assert_dict_contains(actual, exp)
|
||||
|
||||
for key, value in expected.items():
|
||||
attr = getattr(file_group, key)
|
||||
if isinstance(value, dict):
|
||||
assert_dict_contains(attr, value)
|
||||
elif attr != value:
|
||||
raise AssertionError(f'{file_group}.{key} {attr=} != {value}')
|
||||
|
||||
if assert_count is True:
|
||||
if (count := test_session.query(FileGroup).count()) != len(file_groups):
|
||||
raise AssertionError(f'FileGroup count does not match {count} != {len(file_groups)}')
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_files_search(test_client):
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
def _(search_str: str, expected: List[dict]):
|
||||
content = json.dumps({'search_str': search_str})
|
||||
request, response = test_client.post('/api/files/search', content=content)
|
||||
for file_group, exp in zip_longest(response.json['file_groups'], expected):
|
||||
assert_dict_contains(file_group, exp)
|
||||
|
||||
assert len(response.json['file_groups']) == len(expected)
|
||||
|
||||
return _
|
||||
|
||||
|
||||
SINGLEFILE_CONTENTS = '''<html><!--
|
||||
Page saved with SingleFile
|
||||
url: {url}
|
||||
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
|
||||
--><head><meta charset="utf-8">
|
||||
|
||||
<meta name="msapplication-TileColor" content="#2d89ef">
|
||||
<meta name="theme-color" content="#000000">
|
||||
|
||||
{title}
|
||||
|
||||
<body>
|
||||
some body contents
|
||||
</body>
|
||||
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def singlefile_contents_factory():
|
||||
def _(title: str = '', url: str = 'https://example.com') -> str:
|
||||
contents = copy.deepcopy(SINGLEFILE_CONTENTS)
|
||||
contents = contents.format(
|
||||
title=f'<title>{title}</title>',
|
||||
url=url,
|
||||
)
|
||||
return contents
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.Path:
|
||||
archive = test_directory / 'archive'
|
||||
archive.mkdir(exist_ok=True)
|
||||
singlefile = archive / 'singlefile.html'
|
||||
singlefile.write_text(singlefile_contents_factory())
|
||||
return singlefile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def image_bytes_factory():
|
||||
def _() -> bytes:
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
|
||||
Image.new('RGB', (1, 1), color='grey').save(fh)
|
||||
fh.seek(0)
|
||||
image = fh.read()
|
||||
return image
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def insert_file_group(test_session, test_directory):
|
||||
"""Inserts a FileGroup based on the provided paths. Does not verify if the files exist."""
|
||||
|
||||
def _(paths: List[pathlib.Path]):
|
||||
files = [dict(path=str(i), mimetype='fake') for i in paths]
|
||||
params = dict(
|
||||
full_stem=str(test_directory / files_lib.split_path_stem_and_suffix(paths[0])[0]),
|
||||
primary_path=str(paths[0]),
|
||||
files=json.dumps(files),
|
||||
)
|
||||
test_session.execute('INSERT INTO file_group '
|
||||
'(indexed, full_stem, primary_path, files) VALUES '
|
||||
'(true, :full_stem, :primary_path, :files)', params)
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_tags_config(test_directory):
|
||||
from wrolpi.tags import test_tags_config
|
||||
with test_tags_config():
|
||||
(test_directory / 'config').mkdir(exist_ok=True)
|
||||
config_path = test_directory / 'config/tags.yaml'
|
||||
yield config_path
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ class TZDateTime(types.TypeDecorator):
|
|||
def process_bind_param(self, value, dialect):
|
||||
if isinstance(value, datetime):
|
||||
if not value.tzinfo:
|
||||
raise TypeError("tzinfo is required")
|
||||
raise TypeError(f"tzinfo is required: {value}")
|
||||
# Store all timestamps as UTC timezone.
|
||||
value = value.astimezone(timezone.utc)
|
||||
elif isinstance(value, str) and '-' in value:
|
||||
|
|
|
|||
15
wrolpi/db.py
15
wrolpi/db.py
|
|
@ -1,9 +1,13 @@
|
|||
import pathlib
|
||||
import types
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import ContextManager, Tuple, List, Union, Type
|
||||
from typing import ContextManager, Tuple, List, Union, Type, Generator
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extensions
|
||||
import sqlalchemy.exc
|
||||
from psycopg2._psycopg import cursor
|
||||
from sqlalchemy import create_engine, inspect
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
|
@ -166,3 +170,12 @@ def get_ranked_models(ranked_primary_keys: List, model: Type[Base], session: Ses
|
|||
results = list(session.query(model).filter(pkey.in_(ranked_primary_keys)).all())
|
||||
results = sorted(results, key=lambda i: ranked_primary_keys.index(getattr(i, pkey_name)))
|
||||
return results
|
||||
|
||||
|
||||
def mogrify(curs: cursor, values: Union[List, Generator]) -> str:
|
||||
values = list(values) if isinstance(values, types.GeneratorType) else values.copy()
|
||||
count = len(values[0])
|
||||
# If `count = 3`, then s = '(%s,%s,%s)'
|
||||
s = '(' + ','.join("%s" for _ in range(count)) + ')'
|
||||
result = ',\n'.join([curs.mogrify(s, i).decode() for i in values])
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -152,6 +152,14 @@ class UnknownArchive(APIError):
|
|||
pass
|
||||
|
||||
|
||||
class UnknownTag(APIError):
|
||||
pass
|
||||
|
||||
|
||||
class UsedTag(APIError):
|
||||
pass
|
||||
|
||||
|
||||
error_codes = iter(range(1, 1000))
|
||||
|
||||
API_ERRORS = {
|
||||
|
|
@ -334,5 +342,15 @@ API_ERRORS = {
|
|||
'code': next(error_codes),
|
||||
'message': 'Unable to find the archive',
|
||||
'status': HTTPStatus.NOT_FOUND,
|
||||
}
|
||||
},
|
||||
UnknownTag: {
|
||||
'code': next(error_codes),
|
||||
'message': 'Unable to find the tag',
|
||||
'status': HTTPStatus.NOT_FOUND,
|
||||
},
|
||||
UsedTag: {
|
||||
'code': next(error_codes),
|
||||
'message': 'Tag is being used',
|
||||
'status': HTTPStatus.BAD_REQUEST,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,13 +29,17 @@ class Events:
|
|||
def send_global_refresh_modeling_completed(message: str = None):
|
||||
send_event('global_refresh_modeling_completed', message, subject='refresh')
|
||||
|
||||
@staticmethod
|
||||
def send_global_refresh_discovery_completed(message: str = None):
|
||||
send_event('global_refresh_discovery_completed', message, subject='refresh')
|
||||
|
||||
@staticmethod
|
||||
def send_global_refresh_indexing_completed(message: str = None):
|
||||
send_event('global_refresh_indexing_completed', message, subject='refresh')
|
||||
|
||||
@staticmethod
|
||||
def send_global_refresh_delete_completed(message: str = None):
|
||||
send_event('global_refresh_delete_completed', message, subject='refresh')
|
||||
def send_global_after_refresh_completed(message: str = None):
|
||||
send_event('global_after_refresh_completed', message, subject='refresh')
|
||||
|
||||
@staticmethod
|
||||
def send_ready(message: str = None):
|
||||
|
|
|
|||
|
|
@ -1,18 +1,14 @@
|
|||
import pathlib
|
||||
from http import HTTPStatus
|
||||
from typing import List
|
||||
from urllib.request import Request
|
||||
|
||||
from sanic import response
|
||||
from sanic import response, Request
|
||||
from sanic_ext import validate
|
||||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, background_task, get_relative_to_media_directory
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory
|
||||
from wrolpi.errors import InvalidFile
|
||||
from wrolpi.root_api import get_blueprint, json_response
|
||||
from . import lib, schema
|
||||
from ..schema import JSONErrorResponse
|
||||
from ..vars import PYTEST
|
||||
|
||||
bp = get_blueprint('Files', '/api/files')
|
||||
|
||||
|
|
@ -44,38 +40,20 @@ async def delete_file(_: Request, body: schema.DeleteRequest):
|
|||
|
||||
|
||||
@bp.post('/refresh')
|
||||
@openapi.description('Find and index all files in the media directory.')
|
||||
@wrol_mode_check
|
||||
async def refresh(_: Request):
|
||||
await lib.refresh_files()
|
||||
return response.empty()
|
||||
|
||||
|
||||
@bp.post('/refresh/directory')
|
||||
@openapi.description('Find and index all files in the provided directory.')
|
||||
@validate(schema.DirectoryRefreshRequest)
|
||||
@wrol_mode_check
|
||||
async def refresh_directory(_: Request, body: schema.DirectoryRefreshRequest):
|
||||
directory = get_media_directory() / body.directory
|
||||
if PYTEST:
|
||||
await lib.refresh_directory_files_recursively(directory)
|
||||
else:
|
||||
background_task(lib.refresh_directory_files_recursively(directory))
|
||||
return response.empty()
|
||||
|
||||
|
||||
@bp.post('/refresh/list')
|
||||
@openapi.definition(
|
||||
summary='Refresh and index all files in the provided list',
|
||||
body=schema.FilesRefreshListRequest,
|
||||
summary='Refresh and index all paths (files/directories) in the provided list. Refresh all files if not provided.',
|
||||
body=schema.FilesRefreshRequest,
|
||||
)
|
||||
@validate(schema.FilesRefreshListRequest)
|
||||
@wrol_mode_check
|
||||
async def refresh_files_list(_: Request, body: schema.FilesRefreshListRequest):
|
||||
if PYTEST:
|
||||
await lib.refresh_files_list(body.files, body.include_files_near)
|
||||
else:
|
||||
background_task(lib.refresh_files_list(body.files, body.include_files_near))
|
||||
async def refresh(request: Request):
|
||||
paths = None
|
||||
if request.body:
|
||||
media_directory = get_media_directory()
|
||||
if not isinstance(request.json['paths'], list):
|
||||
raise ValueError('Can only refresh a list')
|
||||
|
||||
paths = [media_directory / i for i in request.json['paths']]
|
||||
await lib.refresh_files(paths)
|
||||
return response.empty()
|
||||
|
||||
|
||||
|
|
@ -86,8 +64,9 @@ async def refresh_files_list(_: Request, body: schema.FilesRefreshListRequest):
|
|||
)
|
||||
@validate(schema.FilesSearchRequest)
|
||||
async def post_search_files(_: Request, body: schema.FilesSearchRequest):
|
||||
files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model)
|
||||
return json_response(dict(files=files, totals=dict(files=total)))
|
||||
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
|
||||
body.tag_names)
|
||||
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))
|
||||
|
||||
|
||||
@bp.post('/directories')
|
||||
|
|
@ -106,3 +85,17 @@ def post_directories(_, body: schema.DirectoriesRequest):
|
|||
|
||||
body = {'directories': dirs, 'exists': path.exists(), 'is_dir': path.is_dir(), 'is_file': path.is_file()}
|
||||
return response.json(body)
|
||||
|
||||
|
||||
@bp.post('/tag')
|
||||
@validate(schema.TagFileGroupPost)
|
||||
def post_tag_file_group(_, body: schema.TagFileGroupPost):
|
||||
lib.add_file_group_tag(body.file_group_id, body.tag_name)
|
||||
return response.empty(HTTPStatus.CREATED)
|
||||
|
||||
|
||||
@bp.post('/untag')
|
||||
@validate(schema.TagFileGroupPost)
|
||||
def post_untag_file_group(_, body: schema.TagFileGroupPost):
|
||||
lib.remove_file_group_tag(body.file_group_id, body.tag_name)
|
||||
return response.empty(HTTPStatus.NO_CONTENT)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,18 @@
|
|||
import asyncio
|
||||
import dataclasses
|
||||
import logging
|
||||
import pathlib
|
||||
from typing import Dict, List, Generator, Optional
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import bs4
|
||||
from sqlalchemy import Column, Integer, ForeignKey, String
|
||||
from sqlalchemy.orm import Session, relationship
|
||||
from sqlalchemy import Column, Integer, String, BigInteger, ForeignKey, or_, and_
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from wrolpi.common import register_modeler, ModelHelper, Base, register_after_refresh
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.common import ModelHelper, Base, register_modeler
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
try:
|
||||
import ebooklib
|
||||
|
|
@ -48,10 +49,21 @@ class EBookData:
|
|||
creator: str = None
|
||||
text: str = None
|
||||
title: str = None
|
||||
ebook_path: pathlib.Path = None
|
||||
|
||||
def __bool__(self):
|
||||
return bool(self.cover) or bool(self.creator) or bool(self.text) or bool(self.title)
|
||||
|
||||
def __json__(self):
|
||||
d = {}
|
||||
if self.creator:
|
||||
d['creator'] = self.creator
|
||||
if self.title:
|
||||
d['title'] = self.title
|
||||
if self.ebook_path:
|
||||
d['ebook_path'] = self.ebook_path
|
||||
return d
|
||||
|
||||
|
||||
def extract_text(html: str) -> str:
|
||||
soup = bs4.BeautifulSoup(html, features='html.parser')
|
||||
|
|
@ -59,8 +71,8 @@ def extract_text(html: str) -> str:
|
|||
return text
|
||||
|
||||
|
||||
def mimetype_is_ebook(mimetype: str) -> bool:
|
||||
return any(mimetype.startswith(i) for i in EBOOK_MIMETYPES)
|
||||
def mimetype_is_ebook(mimetype: str) -> List[str]:
|
||||
return [i for i in EBOOK_MIMETYPES if mimetype.startswith(i)]
|
||||
|
||||
|
||||
def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]:
|
||||
|
|
@ -70,7 +82,7 @@ def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]
|
|||
if not mimetype.startswith(EPUB_MIMETYPE):
|
||||
return None
|
||||
|
||||
data = EBookData()
|
||||
data = EBookData(ebook_path=path)
|
||||
|
||||
book = epub.read_epub(path, options=dict(ignore_ncx=True))
|
||||
for key, value in book.metadata.items():
|
||||
|
|
@ -139,53 +151,25 @@ class EBook(ModelHelper, Base):
|
|||
id = Column(Integer, primary_key=True)
|
||||
size: int = Column(Integer)
|
||||
|
||||
title: str = Column(String)
|
||||
creator: str = Column(String)
|
||||
|
||||
ebook_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
ebook_file: File = relationship('File', primaryjoin='EBook.ebook_path==File.path')
|
||||
cover_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
cover_file: File = relationship('File', primaryjoin='EBook.cover_path==File.path')
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
def __repr__(self):
|
||||
cover = 'yes' if self.cover_file else 'no'
|
||||
return f'<EBook id={self.id} path={repr(self.ebook_path)} cover={cover}>'
|
||||
path = str(self.file_group.primary_path)
|
||||
return f'<EBook id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
|
||||
|
||||
def my_paths(self) -> Generator[pathlib.Path, None, None]:
|
||||
if self.ebook_path:
|
||||
yield self.ebook_path
|
||||
if self.cover_path:
|
||||
yield self.cover_path
|
||||
|
||||
def my_files(self) -> Generator[File, None, None]:
|
||||
if self.ebook_file:
|
||||
yield self.ebook_file
|
||||
if self.cover_file:
|
||||
yield self.cover_file
|
||||
|
||||
@staticmethod
|
||||
def find_by_path(path, session) -> Base:
|
||||
return session.query(EBook).filter(EBook.ebook_path == path).one_or_none()
|
||||
|
||||
@staticmethod
|
||||
def find_by_paths(paths: List[pathlib.Path], session) -> List:
|
||||
ebooks = list(session.query(EBook).filter(EBook.ebook_path.in_(paths)))
|
||||
return ebooks
|
||||
|
||||
def __json__(self):
|
||||
d = dict(
|
||||
cover_path=self.cover_path,
|
||||
creator=self.creator,
|
||||
ebook_path=self.ebook_path,
|
||||
id=self.id,
|
||||
size=self.size,
|
||||
title=self.title,
|
||||
)
|
||||
def __json__(self) -> dict:
|
||||
# Convert this eBook's paths to pathlib.Paths.
|
||||
d = self.file_group.__json__()
|
||||
d['data']['ebook_path'] = pathlib.Path(d['data']['ebook_path'])
|
||||
d['data']['cover_path'] = pathlib.Path(d['data']['cover_path']) if 'cover_path' in d['data'] else None
|
||||
return d
|
||||
|
||||
def generate_cover(self) -> Optional[pathlib.Path]:
|
||||
"""Discover this ebook's cover, if found, store it next to the ebook file."""
|
||||
path = self.ebook_file.path if self.ebook_file else self.ebook_path
|
||||
path = self.file_group.primary_path
|
||||
|
||||
if not path:
|
||||
raise ValueError('Cannot generate cover for ebook without ebook file!')
|
||||
|
|
@ -201,113 +185,110 @@ class EBook(ModelHelper, Base):
|
|||
|
||||
logger.warning(f'Unable to generate cover for {self.ebook_path}')
|
||||
|
||||
@property
|
||||
def cover_file(self) -> Optional[dict]:
|
||||
posters = self.file_group.my_poster_files()
|
||||
return posters[0]
|
||||
|
||||
def _model_ebook(ebook: EBook, files: List[File]) -> EBook:
|
||||
@property
|
||||
def cover_path(self) -> Optional[pathlib.Path]:
|
||||
return self.cover_file['path']
|
||||
|
||||
|
||||
def _model_ebook(ebook: EBook) -> EBook:
|
||||
"""Creates an EBook model based off a File. Searches for it's cover in the provided `files`."""
|
||||
# Multiple formats may share this cover.
|
||||
cover_file = next((i for i in files if i.mimetype.split('/')[0] == 'image'), None)
|
||||
epub_file = next(iter(ebook.file_group.my_files('application/epub')), None)
|
||||
mobi_file = next(iter(ebook.file_group.my_files('application/x-mobipocket-ebook')), None)
|
||||
# epub is preferred because it can be indexed.
|
||||
if epub_file:
|
||||
ebook_file = epub_file['path']
|
||||
ebook_mimetype = epub_file['mimetype']
|
||||
elif mobi_file:
|
||||
ebook_file = mobi_file['path']
|
||||
ebook_mimetype = mobi_file['mimetype']
|
||||
else:
|
||||
raise ValueError(f'No epub or mobi found. {ebook}')
|
||||
|
||||
ebook_file = ebook.ebook_file
|
||||
size = ebook_file.path.stat().st_size
|
||||
|
||||
ebook.ebook_file.do_index()
|
||||
size = ebook_file.stat().st_size
|
||||
|
||||
# Only index if it hasn't been done, or if the file has changed.
|
||||
changed = size != ebook.size or cover_file != ebook.cover_file or not ebook.title
|
||||
if not ebook.ebook_file.indexed or changed:
|
||||
changed = size != ebook.size or not ebook.title
|
||||
if epub_file and changed:
|
||||
# Only read the contents of the file if it has changed.
|
||||
try:
|
||||
# Only read the contents of the file if it has changed.
|
||||
data = extract_ebook_data(ebook_file.path, ebook_file.mimetype)
|
||||
data = extract_ebook_data(ebook_file, ebook_mimetype)
|
||||
if data:
|
||||
# Title is a_text.
|
||||
ebook.ebook_file.a_text = ebook.title = data.title
|
||||
ebook.file_group.title = ebook.file_group.a_text = ebook.title = data.title
|
||||
# Creator is b_text.
|
||||
ebook.ebook_file.b_text = ebook.creator = data.creator
|
||||
ebook.file_group.b_text = ebook.creator = data.creator
|
||||
# All text is d_text.
|
||||
ebook.ebook_file.d_text = data.text
|
||||
ebook.file_group.d_text = data.text
|
||||
d: dict = data.__json__()
|
||||
d['ebook_path'] = str(d['ebook_path'])
|
||||
ebook.file_group.data = d
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract book data', exc_info=e)
|
||||
logger.error(f'Failed to extract epub book data', exc_info=e)
|
||||
|
||||
if not ebook.title:
|
||||
if ebook.file_group.my_poster_files():
|
||||
ebook.file_group.data['cover_path'] = ebook.file_group.my_poster_files()[0]['path']
|
||||
else:
|
||||
# Extract cover because there is no cover file.
|
||||
cover_path = ebook.generate_cover()
|
||||
ebook.file_group.append_files(cover_path)
|
||||
ebook.file_group.data['cover_path'] = str(cover_path)
|
||||
|
||||
if not ebook.file_group.title:
|
||||
# Book was not indexed above (probably a MOBI), use the file data.
|
||||
stem, _ = split_path_stem_and_suffix(ebook.ebook_file.path)
|
||||
ebook.title = ebook.ebook_file.a_text = stem
|
||||
stem, _ = split_path_stem_and_suffix(ebook.file_group.primary_path)
|
||||
ebook.file_group.title = stem
|
||||
|
||||
if cover_file:
|
||||
cover_file.associated = True
|
||||
|
||||
ebook.cover_file = cover_file
|
||||
ebook.size = size
|
||||
ebook_file.model = EBook.__tablename__
|
||||
|
||||
return ebook
|
||||
|
||||
|
||||
def find_ebook_files_in_group(group: List[File]):
|
||||
def find_ebook_files_in_group(group: List):
|
||||
"""Returns a list of any File records if they are an ebook."""
|
||||
return list(filter(lambda i: mimetype_is_ebook(i.mimetype), group))
|
||||
|
||||
|
||||
@register_modeler
|
||||
def ebook_modeler(groups: Dict[str, List[File]], session: Session):
|
||||
async def ebook_modeler():
|
||||
"""Searches for ebook files and models them into the ebook table."""
|
||||
# Get all ebook files.
|
||||
ebook_files = {stem: files for stem, group in groups.items() if (files := find_ebook_files_in_group(group))}
|
||||
if not ebook_files:
|
||||
# No ebooks in these groups.
|
||||
return
|
||||
# Get all ebooks records (if any) in one query.
|
||||
ebook_paths = [j.path for i in ebook_files.values() for j in i]
|
||||
ebook_records = {i.ebook_path: i for i in session.query(EBook).filter(EBook.ebook_path.in_(ebook_paths))}
|
||||
|
||||
for stem, ebook_files in ebook_files.items():
|
||||
group = groups[stem]
|
||||
session.flush(group)
|
||||
# Process each ebook file in the group. There may be more than one format in each group, and they may all
|
||||
# share a cover.
|
||||
for ebook_file in ebook_files:
|
||||
ebook: EBook = ebook_records.get(ebook_file.path)
|
||||
if not ebook:
|
||||
ebook = EBook(ebook_file=ebook_file)
|
||||
session.add(ebook)
|
||||
_model_ebook(ebook, group)
|
||||
|
||||
# Remove this group, it will not be processed by other modelers.
|
||||
del groups[stem]
|
||||
|
||||
|
||||
@register_after_refresh
|
||||
def discover_ebook_covers():
|
||||
"""Discovers/extracts ebook covers for any Ebook that is missing one."""
|
||||
with get_db_curs() as curs:
|
||||
stmt = f'''
|
||||
SELECT e.id
|
||||
FROM ebook e
|
||||
LEFT JOIN file f on e.ebook_path = f.path
|
||||
WHERE
|
||||
e.cover_path IS NULL
|
||||
AND f.mimetype LIKE '{EPUB_MIMETYPE}%' '''
|
||||
curs.execute(stmt)
|
||||
missing_covers = [i[0] for i in curs.fetchall()]
|
||||
|
||||
if not missing_covers:
|
||||
logger.info('No ebook covers to generate')
|
||||
return
|
||||
|
||||
logger.warning(f'Discovering/generating {len(missing_covers)} ebook covers')
|
||||
|
||||
for id_ in missing_covers:
|
||||
while True:
|
||||
with get_db_session(commit=True) as session:
|
||||
ebook: EBook = session.query(EBook).filter(EBook.id == id_).one()
|
||||
file_groups = session.query(FileGroup, EBook).filter(
|
||||
or_(
|
||||
FileGroup.mimetype == 'application/epub+zip',
|
||||
FileGroup.mimetype == 'application/x-mobipocket-ebook',
|
||||
),
|
||||
FileGroup.indexed == False,
|
||||
).outerjoin(EBook, and_(EBook.file_group_id == FileGroup.id)) \
|
||||
.limit(10)
|
||||
file_groups: List[Tuple[FileGroup, EBook]] = list(file_groups)
|
||||
|
||||
cover_path = ebook.generate_cover()
|
||||
if cover_path:
|
||||
# Cover was generated or discovered. Associate it with the ebook.
|
||||
cover_file = session.query(File).filter(File.path == cover_path).one_or_none()
|
||||
if not cover_file:
|
||||
cover_file = File(path=cover_path)
|
||||
session.add(cover_file)
|
||||
cover_file.do_index()
|
||||
cover_file.associated = True
|
||||
processed = 0
|
||||
for file_group, ebook in file_groups:
|
||||
processed += 1
|
||||
try:
|
||||
ebook = ebook or EBook(file_group=file_group)
|
||||
if PYTEST:
|
||||
# Handle caching issue during testing.
|
||||
session.expire(file_group)
|
||||
_model_ebook(ebook)
|
||||
session.add(ebook)
|
||||
file_group.model = EBook.__tablename__
|
||||
file_group.indexed = True
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index ebook {file_group}')
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
ebook.cover_file = cover_file
|
||||
if processed < 10:
|
||||
# Did not reach limit, no more books.
|
||||
break
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
|
|
|||
|
|
@ -1,18 +1,17 @@
|
|||
import dataclasses
|
||||
import pathlib
|
||||
from abc import ABC
|
||||
from collections import defaultdict
|
||||
from typing import List, Type, Tuple, Generator
|
||||
from typing import List, Type, Tuple
|
||||
from zipfile import ZipFile
|
||||
|
||||
from wrolpi.vars import PYTEST, FILE_MAX_PDF_SIZE, FILE_MAX_TEXT_SIZE
|
||||
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
|
||||
|
||||
try:
|
||||
from PyPDF2 import PdfReader
|
||||
except ImportError:
|
||||
PdfReader = None
|
||||
|
||||
from wrolpi.common import logger, truncate_generator_bytes, split_lines_by_length, truncate_object_bytes
|
||||
from wrolpi.common import logger, split_lines_by_length, truncate_object_bytes
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -22,16 +21,15 @@ __all__ = ['Indexer', 'DefaultIndexer', 'ZipIndexer', 'TextIndexer', 'find_index
|
|||
class Indexer(object):
|
||||
|
||||
@staticmethod
|
||||
def get_title(file):
|
||||
def get_title(path: pathlib.Path):
|
||||
from wrolpi.files.lib import split_file_name_words
|
||||
path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
words = split_file_name_words(path.name)
|
||||
return words
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file) -> Tuple:
|
||||
def create_index(cls, path: pathlib.Path) -> Tuple:
|
||||
# All files can have their title as the highest priority search.
|
||||
a = cls.get_title(file)
|
||||
a = cls.get_title(path)
|
||||
return a, None, None, None
|
||||
|
||||
|
||||
|
|
@ -43,17 +41,17 @@ class DefaultIndexer(Indexer, ABC):
|
|||
indexer_map = defaultdict(lambda: DefaultIndexer)
|
||||
|
||||
|
||||
def find_indexer(file) -> Type[Indexer]:
|
||||
def find_indexer(mimetype: str) -> Type[Indexer]:
|
||||
"""Find the Indexer for a given File model."""
|
||||
if not file.mimetype:
|
||||
if not mimetype:
|
||||
return DefaultIndexer
|
||||
|
||||
# Get the indexer that matches the mimetype of this file.
|
||||
indexer = next((v for k, v in indexer_map.items() if file.mimetype.startswith(k)), None)
|
||||
indexer = next((v for k, v in indexer_map.items() if mimetype.startswith(k)), None)
|
||||
|
||||
if not indexer:
|
||||
# Use the broad indexer.
|
||||
indexer = indexer_map[file.mimetype.split('/')[0]]
|
||||
indexer = indexer_map[mimetype.split('/')[0]]
|
||||
|
||||
return indexer
|
||||
|
||||
|
|
@ -77,19 +75,18 @@ class ZipIndexer(Indexer, ABC):
|
|||
"""Handles archive files like zip."""
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file):
|
||||
a = cls.get_title(file)
|
||||
file_names = cls.get_file_names(file)
|
||||
def create_index(cls, path: pathlib.Path):
|
||||
a = cls.get_title(path)
|
||||
file_names = cls.get_file_names(path)
|
||||
return a, None, None, file_names
|
||||
|
||||
@classmethod
|
||||
def get_file_names(cls, file) -> List[str]:
|
||||
def get_file_names(cls, path: pathlib.Path) -> List[str]:
|
||||
"""Return all the names of the files in the zip file."""
|
||||
path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
from wrolpi.files.lib import split_file_name_words
|
||||
try:
|
||||
with ZipFile(path, 'r') as zip_:
|
||||
file_names = [' '.join(split_file_name_words(pathlib.Path(name).name)) for name in zip_.namelist()]
|
||||
file_names = ' '.join([split_file_name_words(pathlib.Path(name).name) for name in zip_.namelist()])
|
||||
file_names = truncate_object_bytes(file_names, FILE_MAX_TEXT_SIZE)
|
||||
return file_names
|
||||
except Exception as e:
|
||||
|
|
@ -105,9 +102,8 @@ class TextIndexer(Indexer, ABC):
|
|||
Detects VTT (caption) files and forwards them on."""
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file):
|
||||
path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
a = cls.get_title(file)
|
||||
def create_index(cls, path: pathlib.Path):
|
||||
a = cls.get_title(path)
|
||||
words = cls.get_words(path)
|
||||
return a, None, None, words
|
||||
|
||||
|
|
@ -119,78 +115,3 @@ class TextIndexer(Indexer, ABC):
|
|||
contents = truncate_object_bytes(contents, FILE_MAX_TEXT_SIZE)
|
||||
words = split_lines_by_length(contents)
|
||||
return words
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PDFMetadata:
|
||||
title: str = None
|
||||
author: str = None
|
||||
|
||||
|
||||
@register_indexer('application/pdf')
|
||||
class PDFIndexer(Indexer, ABC):
|
||||
"""Uses PyPDF2 to extract text from a PDF."""
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file):
|
||||
path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
|
||||
file_title = super().get_title(file)
|
||||
|
||||
reader = None
|
||||
if PdfReader is not None:
|
||||
reader = PdfReader(path)
|
||||
|
||||
data = cls.get_pdf_metadata(reader, file)
|
||||
|
||||
if path.stat().st_size > FILE_MAX_PDF_SIZE:
|
||||
logger.warning(f'PDF too large to fully index: {path}')
|
||||
return data.title, data.author, file_title, None
|
||||
|
||||
words = ''
|
||||
try:
|
||||
# PDFs are complex, don't fail to create title index if text extraction fails.
|
||||
words = '\n'.join(truncate_generator_bytes(cls.get_words(reader, path), FILE_MAX_TEXT_SIZE))
|
||||
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
|
||||
words = split_lines_by_length(words)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index {path}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
return data.title, data.author, file_title, words
|
||||
|
||||
@classmethod
|
||||
def get_pdf_metadata(cls, reader: PdfReader, file) -> PDFMetadata:
|
||||
"""Extract title/author from the PDF metadata."""
|
||||
path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
|
||||
data = PDFMetadata()
|
||||
|
||||
if reader is None or reader.metadata is None:
|
||||
logger.error(f'Cannot get title of {path} PyPDF2 is not installed.')
|
||||
data.title = super().get_title(file)
|
||||
return data
|
||||
|
||||
data.title = reader.metadata.title
|
||||
data.author = reader.metadata.author
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def get_words(cls, reader: PdfReader, path: pathlib.Path) -> Generator[str, None, None]:
|
||||
"""
|
||||
Reads all text facing up in a PDF.
|
||||
|
||||
Note, may return more than can be stored in Postgres. See: truncate_generator_bytes
|
||||
"""
|
||||
if PdfReader is None:
|
||||
logger.error(f'Cannot index {path} PyPDF2 is not installed.')
|
||||
yield ''
|
||||
return
|
||||
|
||||
for page in reader.pages:
|
||||
# Get all text facing up.
|
||||
text = page.extract_text(0)
|
||||
# Postgres does not allow null characters.
|
||||
text = text.replace('\x00', '\uFFFD').strip()
|
||||
if text:
|
||||
yield text
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@ import asyncio
|
|||
import datetime
|
||||
import functools
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Union, Dict
|
||||
from typing import List, Tuple, Union, Dict, Generator
|
||||
|
||||
import cachetools.func
|
||||
import psycopg2
|
||||
|
|
@ -16,14 +18,16 @@ from sqlalchemy.orm import Session
|
|||
from wrolpi import flags
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
|
||||
get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name, chunks_by_stem, \
|
||||
partition, ordered_unique_list, chunks, cancelable_wrapper, get_relative_to_media_directory
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
||||
from wrolpi.errors import InvalidFile
|
||||
partition, cancelable_wrapper, \
|
||||
get_files_and_directories, chunks_by_stem, apply_modelers, apply_after_refresh
|
||||
from wrolpi.dates import now, from_timestamp
|
||||
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.vars import PYTEST, FILE_REFRESH_CHUNK_SIZE
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.lang import ISO_639_CODES
|
||||
from wrolpi.tags import TagFile, Tag
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
try:
|
||||
import magic
|
||||
|
|
@ -37,7 +41,7 @@ except ImportError:
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['list_directories_contents', 'delete_file', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
|
||||
'get_mimetype', 'split_file_name_words']
|
||||
'get_mimetype', 'split_file_name_words', 'get_primary_file']
|
||||
|
||||
|
||||
@cachetools.func.ttl_cache(10_000, 30.0)
|
||||
|
|
@ -158,7 +162,7 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
|
|||
return mimetype
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1000)
|
||||
@functools.lru_cache(maxsize=10_000)
|
||||
def get_mimetype(path: Path) -> str:
|
||||
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
|
||||
if magic is None:
|
||||
|
|
@ -177,265 +181,307 @@ def get_mimetype(path: Path) -> str:
|
|||
# Special suffixes within WROLPi.
|
||||
SUFFIXES = {
|
||||
'.info.json',
|
||||
'.en.vtt',
|
||||
'.en.srt',
|
||||
'.readability.json',
|
||||
'.readability.txt',
|
||||
'.readability.html',
|
||||
}
|
||||
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
|
||||
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
|
||||
|
||||
|
||||
@functools.lru_cache()
|
||||
def split_path_stem_and_suffix(path: Union[pathlib.Path, str]) -> Tuple[str, str]:
|
||||
@functools.lru_cache(maxsize=10_000)
|
||||
def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = False) -> Tuple[str, str]:
|
||||
"""Get the path's stem and suffix.
|
||||
|
||||
This function handles WROLPi suffixes like .info.json."""
|
||||
if isinstance(path, str):
|
||||
path = pathlib.Path(path)
|
||||
|
||||
full = str(path) # May or may not be absolute.
|
||||
suffix = next(filter(lambda i: full.endswith(i), SUFFIXES), path.suffix)
|
||||
if suffix:
|
||||
full_ = str(path) # May or may not be absolute.
|
||||
suffix = next(filter(lambda i: full_.endswith(i), SUFFIXES), path.suffix)
|
||||
if suffix and full:
|
||||
return f'{path.parent}/{path.name[:-1 * len(suffix)]}', suffix
|
||||
elif suffix:
|
||||
return path.name[:-1 * len(suffix)], suffix
|
||||
|
||||
# Path has no suffix.
|
||||
return path.name, ''
|
||||
if full:
|
||||
return f'{path.parent}/{path.name}', ''
|
||||
else:
|
||||
return path.name, ''
|
||||
|
||||
|
||||
refresh_logger = logger.getChild('refresh')
|
||||
|
||||
|
||||
@wrol_mode_check
|
||||
async def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime):
|
||||
"""Upsert, index, and model all provided paths."""
|
||||
# Update the idempotency if any files already in the DB.
|
||||
def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
|
||||
"""This generates `FileGroup.files` from a list of files."""
|
||||
files = list()
|
||||
for file in group:
|
||||
mimetype = get_mimetype(file)
|
||||
stat = file.stat()
|
||||
modification_datetime = str(from_timestamp(stat.st_mtime))
|
||||
_, suffix = split_path_stem_and_suffix(file)
|
||||
size = stat.st_size
|
||||
files.append(dict(
|
||||
path=str(file),
|
||||
mimetype=mimetype,
|
||||
modification_datetime=modification_datetime,
|
||||
size=size,
|
||||
suffix=suffix,
|
||||
))
|
||||
return files
|
||||
|
||||
|
||||
def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]], pre_sorted: bool = False) -> pathlib.Path:
|
||||
"""Given a list of files, return the file that we can model or index."""
|
||||
from modules.archive.lib import is_singlefile_file
|
||||
|
||||
if len(files) == 0:
|
||||
raise ValueError(f'Cannot find primary file without any files')
|
||||
elif len(files) == 1:
|
||||
# Only one file is always the primary.
|
||||
return files[0]
|
||||
|
||||
file_mimetypes = [(i, get_mimetype(i)) for i in files]
|
||||
# EPUB has high priority.
|
||||
mimetypes = {i[1] for i in file_mimetypes}
|
||||
has_epub = any(i.startswith('application/epub') for i in mimetypes)
|
||||
for file, mimetype in file_mimetypes:
|
||||
# These are the files that can be modeled.
|
||||
if is_singlefile_file(file):
|
||||
return file
|
||||
if mimetype.startswith('video/'):
|
||||
return file
|
||||
if mimetype.startswith('application/epub'):
|
||||
return file
|
||||
if mimetype.startswith('application/pdf') and not has_epub:
|
||||
return file
|
||||
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
|
||||
# Only return a MOBI if no EPUB is present.
|
||||
return file
|
||||
|
||||
# Secondary file types.
|
||||
for file, mimetype in file_mimetypes:
|
||||
if mimetype.startswith('image/'):
|
||||
return file
|
||||
|
||||
logger.debug(f'Cannot find primary file for group: {files}')
|
||||
|
||||
# Can't find a typical primary file, just use the alphanumeric first file.
|
||||
files = sorted(files) if pre_sorted is False else files
|
||||
file = files[0]
|
||||
return file
|
||||
|
||||
|
||||
def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
|
||||
"""Insert/update all records of the provided files.
|
||||
|
||||
Any inserted files will be marked with `indexed=false`. Any existing files will only be updated if their
|
||||
size/modification_datetime changed.
|
||||
|
||||
It is assumed all files exist."""
|
||||
idempotency = str(idempotency)
|
||||
|
||||
for chunk in chunks_by_stem(files, 100):
|
||||
# Group all files by their shared full-stem. `chunk_by_stem` already sorted the files.
|
||||
grouped = group_files_by_stem(chunk, pre_sorted=True)
|
||||
# Convert the groups into file_groups. Extract common information into `file_group.data`.
|
||||
values = dict()
|
||||
for group in grouped:
|
||||
full_stem, _ = split_path_stem_and_suffix(group[0], full=True)
|
||||
# The primary file is the video/SingleFile/epub, etc.
|
||||
primary_path = get_primary_file(group, pre_sorted=True)
|
||||
# The primary mimetype allows modelers to find its file_groups.
|
||||
mimetype = get_mimetype(primary_path)
|
||||
# The group uses a common modification_datetime so the group will be re-indexed when any of it's files are
|
||||
# modified.
|
||||
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
|
||||
size = sum(i.stat().st_size for i in group)
|
||||
files = json.dumps(_paths_to_files_dict(group))
|
||||
values[full_stem] = (str(primary_path), modification_datetime, mimetype, size, files)
|
||||
values = [(stem, False, idempotency, *i) for stem, i in values.items()]
|
||||
with get_db_curs(commit=True) as curs:
|
||||
values = mogrify(curs, values)
|
||||
stmt = f'''
|
||||
INSERT INTO file_group
|
||||
(full_stem, indexed, idempotency, primary_path, modification_datetime, mimetype, size, files)
|
||||
VALUES {values}
|
||||
ON CONFLICT (full_stem) DO UPDATE
|
||||
SET
|
||||
idempotency=EXCLUDED.idempotency,
|
||||
modification_datetime=EXCLUDED.modification_datetime,
|
||||
mimetype=EXCLUDED.mimetype,
|
||||
size=EXCLUDED.size,
|
||||
files=(
|
||||
-- Do not overwrite files unless the files have changed and need to be re-indexed.
|
||||
CASE
|
||||
WHEN
|
||||
file_group.modification_datetime = EXCLUDED.modification_datetime
|
||||
AND file_group.size = EXCLUDED.size
|
||||
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
|
||||
THEN file_group.files
|
||||
ELSE EXCLUDED.files
|
||||
END
|
||||
),
|
||||
-- Preseve TRUE `indexed` only if the files have not changed.
|
||||
indexed=(
|
||||
file_group.indexed = true
|
||||
AND file_group.modification_datetime = EXCLUDED.modification_datetime
|
||||
AND file_group.size = EXCLUDED.size
|
||||
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
|
||||
)
|
||||
RETURNING id, indexed
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
need_index = [i[0] for i in curs.fetchall() if i[1] is False]
|
||||
if need_index:
|
||||
logger.info(f'Invalidated indexes of {len(need_index)} file groups')
|
||||
logger.debug(f'Invalidated indexes of file groups {need_index}')
|
||||
|
||||
|
||||
async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetime.datetime = None):
|
||||
"""Discover all files in the directories provided in paths, as well as all files in paths.
|
||||
|
||||
All records for files in `paths` that do not exist will be deleted.
|
||||
|
||||
Will refuse to refresh when the media directory is empty."""
|
||||
try:
|
||||
next(get_media_directory().iterdir())
|
||||
except:
|
||||
# We don't want to delete a bunch of files which would exist if the drive was mounted.
|
||||
raise UnknownDirectory(f'Refusing to refresh because media directory is empty or does not exist.')
|
||||
|
||||
if not paths:
|
||||
return
|
||||
raise ValueError('Must provide some paths to refresh.')
|
||||
|
||||
for idx, chunk in enumerate(map(set, chunks_by_stem(paths, FILE_REFRESH_CHUNK_SIZE))):
|
||||
with get_db_session(commit=True) as session:
|
||||
existing_files = list(session.query(File).filter(File.path.in_(chunk)))
|
||||
existing_paths = {i.path for i in existing_files}
|
||||
# Add any new files into the database.
|
||||
new_files = []
|
||||
if new_paths := (chunk - existing_paths):
|
||||
new_files = [File(path=i, idempotency=idempotency) for i in new_paths]
|
||||
session.add_all(new_files)
|
||||
# Apply models to all files. Many files will have no model.
|
||||
all_files = existing_files + new_files
|
||||
apply_modelers(all_files, session)
|
||||
idempotency = idempotency or now()
|
||||
|
||||
# Update idempotency for existing files after modelers. Otherwise SQLAlchemy forgets.
|
||||
for existing_file in existing_files:
|
||||
existing_file.idempotency = idempotency
|
||||
exists, deleted = partition(lambda i: i.exists(), paths)
|
||||
|
||||
if idx > 0:
|
||||
parent = all_files[0].path.parent
|
||||
refresh_logger.debug(f'Committed chunk of {len(chunk)} files. Refreshing near {parent}')
|
||||
# DISCOVER all files, upsert their records.
|
||||
|
||||
# Sleep between chunks to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
async def _refresh_directory_files_recursively(directory: pathlib.Path, idempotency: datetime):
|
||||
"""Refresh all files in a directory, recursively refresh all files in subdirectories."""
|
||||
directories = [directory, ]
|
||||
while directories:
|
||||
directory = directories.pop(0)
|
||||
try:
|
||||
files, new_directories = get_files_and_directories(directory)
|
||||
if exists:
|
||||
# Recursively upsert all files that exist, and files in the existing directories.
|
||||
directories, files = partition(lambda i: i.is_dir(), exists)
|
||||
while directories:
|
||||
directory = directories.pop(0)
|
||||
new_files, new_directories = get_files_and_directories(directory)
|
||||
directories.extend(new_directories)
|
||||
if files:
|
||||
refresh_logger.info(f'Refreshing {len(files)} files in {directory}')
|
||||
await _refresh_files_list(files, idempotency)
|
||||
except Exception as e:
|
||||
refresh_logger.error(f'Failed to refresh files in {directory}', exc_info=e)
|
||||
except asyncio.CancelledError:
|
||||
refresh_logger.error(f'Refresh canceled during {directory}')
|
||||
raise
|
||||
files.extend(new_files)
|
||||
if len(files) >= 100:
|
||||
# Wait until there are enough files to perform the upsert.
|
||||
_upsert_files(files, idempotency)
|
||||
files = list()
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
if files:
|
||||
# Not enough files for the chunks above, finish what is left.
|
||||
_upsert_files(files, idempotency)
|
||||
|
||||
|
||||
@cancelable_wrapper
|
||||
async def apply_indexers():
|
||||
"""Finds all Files that have not been indexed and indexes them."""
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('SELECT path FROM file WHERE indexed = false AND associated = false')
|
||||
missing_index = [pathlib.Path(i[0]) for i in curs.fetchall()]
|
||||
|
||||
refresh_logger.info(f'Indexing {len(missing_index)} files')
|
||||
|
||||
# Indexing can be slow, commit when its been too long.
|
||||
last_commit = now()
|
||||
max_seconds_between_commits = 30
|
||||
|
||||
for chunk in chunks(missing_index, 100):
|
||||
with get_db_session(commit=True) as session:
|
||||
files = session.query(File).filter(File.path.in_(chunk))
|
||||
for file in files:
|
||||
try:
|
||||
file.do_index()
|
||||
except Exception as e:
|
||||
refresh_logger.error(f'Failed to index {file=}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
if (now() - last_commit).total_seconds() > max_seconds_between_commits:
|
||||
refresh_logger.debug('Committing because its been too long.')
|
||||
session.commit()
|
||||
last_commit = now()
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
last_commit = now()
|
||||
|
||||
refresh_logger.info(f'Indexed chunk of {len(chunk)} files')
|
||||
|
||||
|
||||
@wrol_mode_check
|
||||
async def refresh_files_list(paths: List[str], include_files_near: bool = True):
|
||||
"""Refresh a list of files. Will not refresh directories or subdirectories.
|
||||
|
||||
Parameters:
|
||||
paths: A list of files that need to be refreshed. If not absolute, they are assumed to be relative to the
|
||||
media directory.
|
||||
include_files_near: If true, will also refresh files that share the stem of the provided files.
|
||||
"""
|
||||
refresh_logger.info(f'Refreshing {len(paths)} files list')
|
||||
|
||||
media_directory = get_media_directory()
|
||||
|
||||
# Convert all paths to absolute paths in the media directory.
|
||||
paths = [pathlib.Path(i) for i in paths]
|
||||
paths = [media_directory / i if not i.is_absolute() else i for i in paths]
|
||||
|
||||
if not paths:
|
||||
raise FileNotFoundError('No files to refresh')
|
||||
|
||||
if include_files_near:
|
||||
new_paths = []
|
||||
for path in paths:
|
||||
new_paths.extend(glob_shared_stem(path))
|
||||
paths.extend(new_paths)
|
||||
|
||||
paths, deleted_paths = partition(lambda i: i.exists(), paths)
|
||||
|
||||
idempotency = now()
|
||||
await _refresh_files_list(paths, idempotency)
|
||||
|
||||
# Delete any files near the paths that no longer exist.
|
||||
paths_stems = [split_path_stem_and_suffix(i)[0] for i in paths]
|
||||
paths_stems = [str(media_directory / i) for i in paths_stems]
|
||||
with get_db_curs(commit=True) as curs:
|
||||
# Get all paths that match the stems of these refreshed files.
|
||||
params = dict(paths_stems=paths_stems, idempotency=idempotency)
|
||||
curs.execute('''
|
||||
UPDATE file SET idempotency = %(idempotency)s
|
||||
WHERE full_stem = ANY (%(paths_stems)s)
|
||||
RETURNING path
|
||||
''', params)
|
||||
existing_paths = [pathlib.Path(i[0]) for i in curs.fetchall()]
|
||||
# Delete any files which no longer exist.
|
||||
deleted_paths.extend([i for i in existing_paths if not i.exists()])
|
||||
deleted_paths = list(map(str, deleted_paths))
|
||||
curs.execute('DELETE FROM file WHERE path = ANY(%(deleted_paths)s)', dict(deleted_paths=deleted_paths))
|
||||
|
||||
apply_after_refresh()
|
||||
await apply_indexers()
|
||||
|
||||
refresh_logger.info('Done refreshing files list')
|
||||
wheres = ''
|
||||
if paths:
|
||||
# Use LIKE to delete any children of directories that are deleted.
|
||||
wheres = ' OR '.join([curs.mogrify('full_stem LIKE %s', (f'{i}/%',)).decode() for i in paths])
|
||||
deleted_files = ''
|
||||
if deleted:
|
||||
# Delete any paths that do not exist.
|
||||
deleted_files = ' OR '.join([curs.mogrify('full_stem = %s', (str(i),)).decode() for i in deleted])
|
||||
idempotency = curs.mogrify('%s', (idempotency,)).decode()
|
||||
wheres = f' ( (idempotency != {idempotency} OR idempotency is null) AND ({wheres}))' if wheres else ''
|
||||
stmt = f'''
|
||||
DELETE FROM file_group
|
||||
WHERE
|
||||
-- Delete all known-deleted files.
|
||||
{deleted_files}
|
||||
-- Delete any files in the refreshed paths that were not updated.
|
||||
{" OR " + wheres if deleted_files else wheres}
|
||||
'''
|
||||
logger.debug(stmt)
|
||||
curs.execute(stmt)
|
||||
|
||||
|
||||
@limit_concurrent(1) # Only one refresh at a time.
|
||||
@wrol_mode_check
|
||||
@cancelable_wrapper
|
||||
async def refresh_files():
|
||||
async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = True):
|
||||
"""Find, model, and index all files in the media directory."""
|
||||
with flags.refreshing:
|
||||
refresh_logger.warning('Refreshing all files')
|
||||
Events.send_global_refresh_started()
|
||||
if isinstance(paths, str):
|
||||
paths = [pathlib.Path(paths), ]
|
||||
if isinstance(paths, pathlib.Path):
|
||||
paths = [paths, ]
|
||||
|
||||
# TODO remove this later when everyone has migrated their files.
|
||||
from modules.archive.lib import migrate_archive_files
|
||||
migrate_archive_files()
|
||||
with flags.refreshing:
|
||||
if not paths:
|
||||
refresh_logger.warning('Refreshing all files')
|
||||
else:
|
||||
refresh_logger.warning(f'Refreshing {", ".join(list(map(str, paths)))}')
|
||||
if send_events:
|
||||
Events.send_global_refresh_started()
|
||||
|
||||
idempotency = now()
|
||||
|
||||
# Add all files in the media directory to the DB.
|
||||
await _refresh_directory_files_recursively(get_media_directory(), idempotency)
|
||||
paths = paths or [get_media_directory()]
|
||||
await refresh_discover_paths(paths, idempotency)
|
||||
if send_events:
|
||||
Events.send_global_refresh_discovery_completed()
|
||||
|
||||
Events.send_global_refresh_modeling_completed()
|
||||
# Model all files that have not been indexed.
|
||||
await apply_modelers()
|
||||
if send_events:
|
||||
Events.send_global_refresh_modeling_completed()
|
||||
|
||||
# Remove any records where the file no longer exists.
|
||||
with get_db_curs(commit=True) as curs:
|
||||
curs.execute('DELETE FROM file WHERE idempotency < %s OR idempotency is null RETURNING path',
|
||||
(idempotency,))
|
||||
deleted = list(curs.fetchall())
|
||||
logger.debug(f'{deleted=}')
|
||||
logger.warning(f'Removed {len(deleted)} missing files')
|
||||
|
||||
Events.send_global_refresh_delete_completed()
|
||||
|
||||
apply_after_refresh()
|
||||
# Index the rest of the files that were not indexed by modelers.
|
||||
await apply_indexers()
|
||||
Events.send_global_refresh_indexing_completed()
|
||||
if send_events:
|
||||
Events.send_global_refresh_indexing_completed()
|
||||
|
||||
Events.send_global_refresh_completed()
|
||||
# Cleanup any outdated file data.
|
||||
await apply_after_refresh()
|
||||
if send_events:
|
||||
Events.send_global_after_refresh_completed()
|
||||
|
||||
if send_events:
|
||||
Events.send_global_refresh_completed()
|
||||
refresh_logger.warning('Done refreshing Files')
|
||||
|
||||
flags.refresh_complete.set()
|
||||
|
||||
|
||||
@limit_concurrent(1)
|
||||
@wrol_mode_check
|
||||
@cancelable_wrapper
|
||||
async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str], send_events: bool = True):
|
||||
"""Upsert and index all files within a directory (recursively).
|
||||
async def apply_indexers():
|
||||
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
while True:
|
||||
# Continually query for Files that have not been indexed.
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20)
|
||||
file_groups: List[FileGroup] = list(file_groups)
|
||||
|
||||
Any records of the files that are no longer in the directory will be removed."""
|
||||
if isinstance(directory, str):
|
||||
directory = pathlib.Path(directory)
|
||||
if directory.is_file():
|
||||
raise ValueError(f'Cannot refresh files of a file: {directory=}')
|
||||
processed = 0
|
||||
for file_group in file_groups:
|
||||
processed += 1
|
||||
try:
|
||||
file_group.do_index()
|
||||
except Exception:
|
||||
# Error has already been logged in .do_index.
|
||||
if PYTEST:
|
||||
raise
|
||||
# Always mark the FileGroup as indexed. We won't try to index it again.
|
||||
file_group.indexed = True
|
||||
|
||||
with flags.refreshing_directory:
|
||||
relative_path = str(get_relative_to_media_directory(directory))
|
||||
if send_events:
|
||||
Events.send_directory_refresh_started(f'Refresh of {repr(relative_path)} has started.')
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# All Files older than this will be removed.
|
||||
idempotency = now()
|
||||
|
||||
refresh_logger.info(f'Recursively refreshing all files in {directory}')
|
||||
|
||||
await _refresh_directory_files_recursively(directory, idempotency)
|
||||
|
||||
# Remove any records where the file no longer exists.
|
||||
with get_db_curs(commit=True) as curs:
|
||||
refresh_logger.debug(f'Deleting files no longer in {directory}')
|
||||
params = dict(
|
||||
directory=f'{directory}/%', # trailing / is important!
|
||||
idempotency=idempotency,
|
||||
)
|
||||
stmt = '''
|
||||
DELETE FROM file
|
||||
WHERE
|
||||
(idempotency < %(idempotency)s OR idempotency IS NULL)
|
||||
AND path LIKE %(directory)s'''
|
||||
curs.execute(stmt, params)
|
||||
|
||||
apply_after_refresh()
|
||||
await apply_indexers()
|
||||
|
||||
if send_events:
|
||||
Events.send_directory_refresh_completed(f'Refresh of {repr(relative_path)} has completed.')
|
||||
refresh_logger.info(f'Done refreshing files in {directory}')
|
||||
if processed < 20:
|
||||
# Processed less than the limit, don't do the next query.
|
||||
break
|
||||
|
||||
|
||||
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None) -> \
|
||||
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
|
||||
tags: List[str] = None) -> \
|
||||
Tuple[List[dict], int]:
|
||||
"""Search the Files table.
|
||||
"""Search the FileGroup table.
|
||||
|
||||
Order the returned Files by their rank if `search_str` is provided. Return all files if
|
||||
`search_str` is empty.
|
||||
|
|
@ -448,9 +494,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
model: Only return files that match this model.
|
||||
"""
|
||||
params = dict(offset=offset, limit=limit)
|
||||
wheres = ['associated = false']
|
||||
wheres = []
|
||||
selects = []
|
||||
order_by = '1 ASC'
|
||||
joins = []
|
||||
|
||||
if search_str:
|
||||
params['search_str'] = search_str
|
||||
|
|
@ -474,25 +521,54 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
params['model'] = model
|
||||
wheres.append('model = %(model)s')
|
||||
|
||||
if tags:
|
||||
where_, params_, join_ = tag_names_to_clauses(tags)
|
||||
wheres.append(where_)
|
||||
params.update(params_)
|
||||
joins.append(join_)
|
||||
|
||||
wheres = '\n AND '.join(wheres)
|
||||
selects = f"{', '.join(selects)}, " if selects else ""
|
||||
join = '\n'.join(joins)
|
||||
stmt = f'''
|
||||
SELECT path, {selects} COUNT(*) OVER() AS total
|
||||
FROM file
|
||||
SELECT fg.id, {selects} COUNT(*) OVER() AS total
|
||||
FROM file_group fg
|
||||
{join}
|
||||
{f"WHERE {wheres}" if wheres else ""}
|
||||
GROUP BY fg.id
|
||||
ORDER BY {order_by}
|
||||
OFFSET %(offset)s LIMIT %(limit)s
|
||||
'''
|
||||
logger.debug(stmt)
|
||||
|
||||
results, total = handle_search_results(stmt, params)
|
||||
results, total = handle_file_group_search_results(stmt, params)
|
||||
return results, total
|
||||
|
||||
|
||||
def handle_search_results(statement: str, params):
|
||||
def tag_names_to_clauses(tags: List[str]):
|
||||
"""Create the SQL necessary to filter the `file_group` table by the provided Tag names."""
|
||||
params = dict()
|
||||
|
||||
if not tags:
|
||||
return '', params, ''
|
||||
|
||||
where_tags = []
|
||||
for idx, tag_name in enumerate(tags):
|
||||
where_tags.append(f't.name = %(tag_name{idx})s')
|
||||
params[f'tag_name{idx}'] = tag_name
|
||||
where_tags = ' OR '.join(where_tags)
|
||||
wheres = f'({where_tags})'
|
||||
|
||||
join = '''LEFT JOIN tag_file tf ON tf.file_group_id = fg.id
|
||||
LEFT JOIN tag t ON t.id = tf.tag_id'''
|
||||
return wheres, params, join
|
||||
|
||||
|
||||
def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List[dict], int]:
|
||||
"""
|
||||
Execute the provided SQL statement and fetch the Files returned.
|
||||
|
||||
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to search.
|
||||
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search.
|
||||
|
||||
See: `search`
|
||||
"""
|
||||
|
|
@ -504,25 +580,30 @@ def handle_search_results(statement: str, params):
|
|||
# No videos
|
||||
return [], 0
|
||||
total = results[0]['total'] if results else 0
|
||||
ranked_paths = [pathlib.Path(i['path']) for i in results]
|
||||
ordered_ids = [i['id'] for i in results]
|
||||
try:
|
||||
ranks = [i['ts_rank'] for i in results]
|
||||
except KeyError:
|
||||
# No `ts_rank`, probably not searching `file.textsearch`.
|
||||
# No `ts_rank`, probably not searching `file_group.textsearch`.
|
||||
ranks = []
|
||||
|
||||
with get_db_session() as session:
|
||||
results = get_ranked_models(ranked_paths, File, session=session)
|
||||
try:
|
||||
results = assign_file_prefetched_models(results)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to assign file prefetched models', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
results = [i.__json__() for i in results]
|
||||
# Preserve the ts_ranks, if any.
|
||||
for idx, rank in enumerate(ranks):
|
||||
results[idx]['ts_rank'] = rank
|
||||
from modules.videos.models import Video
|
||||
results = session.query(FileGroup, Video) \
|
||||
.filter(FileGroup.id.in_(ordered_ids)) \
|
||||
.outerjoin(Video, Video.file_group_id == FileGroup.id)
|
||||
# Order FileGroups by their location in ordered_ids.
|
||||
file_groups: List[Tuple[FileGroup, Video]] = sorted(results, key=lambda i: ordered_ids.index(i[0].id))
|
||||
results = list()
|
||||
for rank, (file_group, video) in zip_longest(ranks, file_groups):
|
||||
video: Video
|
||||
if video:
|
||||
results.append(video.__json__())
|
||||
else:
|
||||
results.append(file_group.__json__())
|
||||
# Preserve the ts_ranks, if any.
|
||||
if rank:
|
||||
results[-1]['ts_rank'] = rank
|
||||
|
||||
return results, total
|
||||
|
||||
|
|
@ -544,50 +625,6 @@ def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]:
|
|||
return paths
|
||||
|
||||
|
||||
def assign_file_prefetched_models(files: List[File]) -> List[File]:
|
||||
"""
|
||||
Prefetch the sub-models (matching `File.model`) in bulk to avoid issuing a query for every File.
|
||||
|
||||
>>> f = File(path='something', model='video')
|
||||
>>> v = Video(video_path='something')
|
||||
>>> results = assign_file_prefetched_models([f])
|
||||
>>> results[0]
|
||||
File(path='something', model='video')
|
||||
>>> results[0].prefetched_model == v
|
||||
True
|
||||
"""
|
||||
if not files:
|
||||
return files
|
||||
|
||||
files = files.copy()
|
||||
files_index = {i.path: files.index(i) for i in files}
|
||||
session = Session.object_session(files[0])
|
||||
|
||||
files_by_model = dict()
|
||||
for file in files:
|
||||
if not file.model:
|
||||
continue
|
||||
try:
|
||||
files_by_model[file.model].append(file)
|
||||
except KeyError:
|
||||
files_by_model[file.model] = [file, ]
|
||||
|
||||
logger.debug(f'Prefetching file models: {list(files_by_model.keys())}')
|
||||
table_map = {i: get_model_by_table_name(i) for i in {j.model for j in files}}
|
||||
for model, model_files in files_by_model.items():
|
||||
table = table_map[model]
|
||||
try:
|
||||
sub_models = table.find_by_paths([i.path for i in model_files], session)
|
||||
for sub_model in sub_models:
|
||||
file: File = files[files_index[sub_model.primary_path]]
|
||||
file.prefetched_model = sub_model
|
||||
except NotImplementedError:
|
||||
# Model has not defined the necessary methods.
|
||||
pass
|
||||
|
||||
return files
|
||||
|
||||
|
||||
def get_matching_directories(path: Union[str, Path]) -> List[str]:
|
||||
"""
|
||||
Return a list of directory strings that start with the provided path. If the path is a directory, return it's
|
||||
|
|
@ -618,7 +655,7 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
|
|||
WHITESPACE = re.compile(r'[\s_]')
|
||||
|
||||
|
||||
def split_file_name_words(name: str) -> List[str]:
|
||||
def split_file_name_words(name: str) -> str:
|
||||
"""Split words in a filename.
|
||||
|
||||
Words are assumed to be separated by underscore, dash, or space. Words with a dash are included as a group, and
|
||||
|
|
@ -641,34 +678,83 @@ def split_file_name_words(name: str) -> List[str]:
|
|||
if suffix:
|
||||
words.append(suffix.lstrip('.'))
|
||||
|
||||
words = ordered_unique_list(words)
|
||||
words = ' '.join(words)
|
||||
return words
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to split filename into words: {name}', exc_info=e)
|
||||
return [name]
|
||||
return name
|
||||
|
||||
|
||||
def get_file_statistics():
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('''
|
||||
SELECT
|
||||
COUNT(path) AS "total_count",
|
||||
COUNT(path) FILTER (WHERE file.mimetype = 'application/pdf') AS "pdf_count",
|
||||
COUNT(path) FILTER (WHERE file.mimetype = 'application/zip') AS "zip_count",
|
||||
COUNT(path) FILTER (WHERE file.mimetype LIKE 'video/%') AS "video_count",
|
||||
COUNT(path) FILTER (WHERE file.mimetype LIKE 'image/%' AND file.associated = FALSE) AS "image_count",
|
||||
COUNT(path) FILTER (WHERE file.mimetype LIKE 'audio/%' AND file.associated = FALSE) AS "audio_count",
|
||||
SUM(size)::BIGINT AS "total_size"
|
||||
COUNT(id) AS "total_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype = 'application/x-mobipocket-ebook') AS "ebook_count",
|
||||
SUM(size)::BIGINT AS "total_size",
|
||||
(SELECT COUNT(*) FROM archive) AS archive_count
|
||||
FROM
|
||||
file
|
||||
file_group
|
||||
''')
|
||||
statistics = dict(curs.fetchall()[0])
|
||||
statistics['total_size'] = statistics['total_size'] or 0
|
||||
|
||||
curs.execute('SELECT COUNT(*) FROM archive')
|
||||
statistics['archive_count'] = curs.fetchall()[0][0]
|
||||
|
||||
curs.execute('SELECT COUNT(*) FROM ebook')
|
||||
statistics['ebook_count'] = curs.fetchall()[0][0]
|
||||
|
||||
return statistics
|
||||
|
||||
|
||||
def group_files_by_stem(files: List[pathlib.Path], pre_sorted: bool = False) -> \
|
||||
Generator[List[pathlib.Path], None, None]:
|
||||
"""
|
||||
Return lists of paths, each list contains only paths that share a common stem.
|
||||
|
||||
>>> a = ['foo.txt', 'foo.mp4', 'bar.txt']
|
||||
>>> group_files_by_stem(list(map((pathlib.Path, a))))
|
||||
# Generator(['foo.mp4', 'foo.txt'], ['bar.txt'])
|
||||
|
||||
@param files: A list of paths to be grouped. All paths must be in the same directory!
|
||||
@param pre_sorted: This function requires the files to be sorted, it will sort them by default.
|
||||
"""
|
||||
files = sorted(files) if not pre_sorted else files
|
||||
file = files.pop(0)
|
||||
group = [file, ]
|
||||
prev_stem, _ = split_path_stem_and_suffix(file)
|
||||
for file in files:
|
||||
stem, suffix = split_path_stem_and_suffix(file)
|
||||
if stem == prev_stem:
|
||||
group.append(file)
|
||||
continue
|
||||
# Stem has changed, group is finished.
|
||||
yield group
|
||||
group = [file, ]
|
||||
prev_stem = stem
|
||||
yield group
|
||||
|
||||
|
||||
def _get_tag(file_group_id: int, tag_name: str, session: Session):
|
||||
file_group: FileGroup = session.query(FileGroup).filter_by(id=file_group_id).one_or_none()
|
||||
if not file_group:
|
||||
raise UnknownFile(f'Cannot find FileGroup with id {file_group_id}')
|
||||
|
||||
tag: Tag = Tag.find_by_name(tag_name, session)
|
||||
if not tag:
|
||||
raise UnknownTag(f'Cannot find Tag with name {tag_name}')
|
||||
|
||||
return file_group, tag
|
||||
|
||||
|
||||
@optional_session(commit=True)
|
||||
def add_file_group_tag(file_group_id: int, tag_name: str, session: Session = None) -> TagFile:
|
||||
file_group, tag = _get_tag(file_group_id, tag_name, session)
|
||||
tag_file = file_group.add_tag(tag, session)
|
||||
return tag_file
|
||||
|
||||
|
||||
@optional_session(commit=True)
|
||||
def remove_file_group_tag(file_group_id: int, tag_name: str, session: Session = None):
|
||||
file_group, tag = _get_tag(file_group_id, tag_name, session)
|
||||
file_group.remove_tag(tag, session)
|
||||
|
|
|
|||
|
|
@ -1,36 +1,81 @@
|
|||
import pathlib
|
||||
from datetime import datetime
|
||||
from typing import List, Type, Optional
|
||||
|
||||
from sqlalchemy import Integer, Column, String, Computed, Boolean
|
||||
from sqlalchemy.orm import deferred, Session, validates
|
||||
from sqlalchemy import Column, String, Computed, BigInteger, Boolean
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.orm import deferred, relationship, Session
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
import wrolpi.files.indexers
|
||||
from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector, logger, get_model_by_table_name, \
|
||||
truncate_object_bytes
|
||||
from wrolpi.dates import TZDateTime, from_timestamp, now
|
||||
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory
|
||||
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.files import indexers
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
|
||||
from wrolpi.tags import Tag, TagFile
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
class File(ModelHelper, Base):
|
||||
"""A representation of a file on disk.
|
||||
def into_db(obj):
|
||||
if isinstance(obj, datetime):
|
||||
return strftime(obj)
|
||||
if isinstance(obj, pathlib.Path):
|
||||
return str(obj)
|
||||
return obj
|
||||
|
||||
Can be searched using `textsearch` once the data has been populated by the respective Indexer.
|
||||
|
||||
def out_of_db(media_directory, obj):
|
||||
if obj and isinstance(obj, str) and obj.startswith(media_directory):
|
||||
return pathlib.Path(obj)
|
||||
if obj and isinstance(obj, str) and obj[0].isdigit():
|
||||
try:
|
||||
return strptime_ms(obj)
|
||||
except Exception:
|
||||
# Wasn't a datetime after all.
|
||||
pass
|
||||
return obj
|
||||
|
||||
|
||||
class FancyJSON(types.TypeDecorator):
|
||||
"""Converts pathlib.Path to strings when moving into DB, and vice versa.
|
||||
|
||||
Converts datetime to ISO strings when moving into DB, and vice versa.
|
||||
"""
|
||||
__tablename__ = 'file'
|
||||
path: pathlib.Path = Column(MediaPathType, primary_key=True)
|
||||
impl = types.JSON
|
||||
|
||||
associated = Column(Boolean, default=False)
|
||||
full_stem = Column(String) # f'{directory}/{stem}'
|
||||
idempotency = Column(TZDateTime, default=lambda: now())
|
||||
indexed = Column(Boolean, default=False)
|
||||
def process_bind_param(self, value, dialect):
|
||||
if value:
|
||||
# datetime(2020, 1, 1, 0, 0, 0) <-> '2020-01-01T00:00:00'
|
||||
value = recursive_map(value, into_db)
|
||||
return value
|
||||
|
||||
def process_result_value(self, value, dialect) -> None:
|
||||
if value:
|
||||
media_directory = str(get_media_directory())
|
||||
value = recursive_map(value, lambda i: out_of_db(media_directory, i))
|
||||
return value
|
||||
|
||||
|
||||
class FileGroup(ModelHelper, Base):
|
||||
__tablename__ = 'file_group'
|
||||
id: int = Column(BigInteger, primary_key=True)
|
||||
|
||||
data = Column(FancyJSON) # populated by the modeler
|
||||
files = Column(FancyJSON, nullable=False) # populated during discovery
|
||||
full_stem = Column(String, unique=True)
|
||||
idempotency = Column(TZDateTime)
|
||||
indexed = Column(Boolean, default=lambda: False, nullable=False)
|
||||
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
|
||||
model = Column(String) # Filled out by the modeler. Will be the table name (video, archive, etc.).
|
||||
model = Column(String) # "video", "archive", "ebook", etc.
|
||||
modification_datetime = Column(TZDateTime)
|
||||
size = Column(Integer)
|
||||
suffix = Column(String) # aka the extension.
|
||||
title = Column(String) # Provided by the modeler, or it will be the entire file name (with suffix).
|
||||
# the Path of the file that can be modeled or indexed.
|
||||
primary_path = Column(MediaPathType, nullable=False, unique=True)
|
||||
size = Column(BigInteger, default=lambda: 0)
|
||||
title = Column(String)
|
||||
|
||||
tag_files: InstrumentedList = relationship('TagFile')
|
||||
|
||||
a_text = deferred(Column(String))
|
||||
b_text = deferred(Column(String))
|
||||
|
|
@ -44,121 +89,153 @@ class File(ModelHelper, Base):
|
|||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
|
||||
''')))
|
||||
|
||||
prefetched_model = None # This may be filled with record matching `model`.
|
||||
|
||||
def __repr__(self):
|
||||
path = str(self.path.relative_to(get_media_directory()))
|
||||
return f'<File {path=} mime={self.mimetype} model={self.model}>'
|
||||
|
||||
def __json__(self) -> dict:
|
||||
path = self.path.relative_to(get_media_directory())
|
||||
d = dict(
|
||||
associated=self.associated,
|
||||
directory=self.path.parent,
|
||||
full_stem=self.full_stem,
|
||||
key=path, # React browser expects this.
|
||||
mimetype=self.mimetype,
|
||||
model=self.model,
|
||||
modified=self.modification_datetime, # React browser expects this.
|
||||
path=path,
|
||||
size=self.size,
|
||||
suffix=self.suffix,
|
||||
title=self.title,
|
||||
)
|
||||
if self.model and self.prefetched_model:
|
||||
d[self.model] = self.prefetched_model.__json__()
|
||||
elif self.model:
|
||||
model: ModelHelper = get_model_by_table_name(self.model)
|
||||
session = Session.object_session(self)
|
||||
instance = model.find_by_path(self.path, session)
|
||||
if not instance:
|
||||
logger.warning(f'Could not find instance of {model} for {self.path}')
|
||||
return d
|
||||
d[self.model] = instance.__json__()
|
||||
m = f'model={self.model}' if self.model else f'mimetype={self.mimetype}'
|
||||
return f'<FileGroup id={self.id} {m} primary_path={repr(str(self.primary_path))}>'
|
||||
|
||||
def __json__(self):
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
_, suffix = split_path_stem_and_suffix(self.primary_path)
|
||||
tags = sorted([i.tag.name for i in self.tag_files])
|
||||
d = {
|
||||
'data': self.data,
|
||||
'directory': self.primary_path.parent,
|
||||
'files': self.my_files(),
|
||||
'full_stem': pathlib.Path(self.full_stem) if self.full_stem else None,
|
||||
'id': self.id,
|
||||
'mimetype': self.mimetype,
|
||||
'model': self.model,
|
||||
'modified': self.modification_datetime or None,
|
||||
'name': self.primary_path.name,
|
||||
'primary_path': self.primary_path,
|
||||
'size': self.size,
|
||||
'suffix': suffix,
|
||||
'tags': tags,
|
||||
'title': self.title,
|
||||
'key': self.primary_path,
|
||||
}
|
||||
return d
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, File):
|
||||
return other.path == self.path
|
||||
if isinstance(other, pathlib.Path):
|
||||
return other == self.path
|
||||
if isinstance(other, str):
|
||||
return other == str(self.path)
|
||||
return False
|
||||
@property
|
||||
def name(self) -> str:
|
||||
if self.title:
|
||||
return self.title
|
||||
return self.primary_path.name
|
||||
|
||||
@optional_session
|
||||
def add_tag(self, tag: Tag, session: Session = None) -> TagFile:
|
||||
return tag.add_tag(self, session)
|
||||
|
||||
@optional_session
|
||||
def remove_tag(self, tag: Tag, session: Session = None):
|
||||
tag.remove_tag(self, session)
|
||||
|
||||
def append_files(self, *paths: pathlib.Path):
|
||||
"""Add all `paths` to this FileGroup.files."""
|
||||
from wrolpi.files.lib import get_mimetype
|
||||
new_files = list(self.files) if self.files else list()
|
||||
for file in paths:
|
||||
new_files.append(dict(path=file, mimetype=get_mimetype(file)))
|
||||
self.files = new_files
|
||||
|
||||
def my_files(self, *mimetypes: str) -> List[dict]:
|
||||
"""Return all files related to this group that match any of the provided mimetypes.
|
||||
|
||||
>>> FileGroup().my_files()
|
||||
>>> FileGroup().my_files('application/pdf')
|
||||
>>> FileGroup().my_files('video/')
|
||||
"""
|
||||
files = self.files
|
||||
if not files:
|
||||
logger.error(f'{self} has no files!')
|
||||
raise ValueError(f'{self} has no files!')
|
||||
|
||||
# Convert path strings to Paths.
|
||||
for i in range(len(files)):
|
||||
files[i]['path'] = pathlib.Path(files[i]['path'])
|
||||
|
||||
if mimetypes:
|
||||
files = list(filter(lambda i: any(i['mimetype'].startswith(m) for m in mimetypes), files))
|
||||
|
||||
# Sort files to avoid random order.
|
||||
return sorted(files, key=lambda i: i['path'])
|
||||
|
||||
def my_paths(self, *mimetypes: str) -> List[pathlib.Path]:
|
||||
return [i['path'] for i in self.my_files(*mimetypes)]
|
||||
|
||||
def my_video_files(self):
|
||||
"""Return all my Files that are videos."""
|
||||
return self.my_files('video/')
|
||||
|
||||
def my_json_files(self):
|
||||
"""Return all my Files that are JSON."""
|
||||
return self.my_files('application/json')
|
||||
|
||||
def my_poster_files(self) -> List[dict]:
|
||||
"""Return all my Files that are images."""
|
||||
return self.my_files('image/')
|
||||
|
||||
def my_subtitle_files(self) -> List[dict]:
|
||||
"""Return all my Files that have text/srt or text/vtt mimetype."""
|
||||
return self.my_files('text/srt', 'text/vtt')
|
||||
|
||||
def my_text_files(self) -> List[dict]:
|
||||
"""Return all my Files that have a text mimetype. But, do not include subtitle files."""
|
||||
text_files = self.my_files('text/')
|
||||
subtitle_paths = [i['path'] for i in self.my_subtitle_files()]
|
||||
return [i for i in text_files if i['path'] not in subtitle_paths]
|
||||
|
||||
def my_ebook_files(self) -> List[dict]:
|
||||
return self.my_files('application/epub', 'application/x-mobipocket-ebook')
|
||||
|
||||
def delete(self):
|
||||
"""Delete this FileGroup record, and all of its files."""
|
||||
for path in self.my_paths():
|
||||
path.unlink()
|
||||
|
||||
session = Session.object_session(self)
|
||||
if session:
|
||||
session.delete(self)
|
||||
|
||||
@property
|
||||
def indexer(self):
|
||||
from wrolpi.files import lib
|
||||
return wrolpi.files.indexers.find_indexer(self)
|
||||
|
||||
def do_index(self, force_index: bool = False):
|
||||
"""Gather any missing information about this file. Index the contents of this file using an Indexer."""
|
||||
def indexer(self) -> Type[indexers.Indexer]:
|
||||
if not self.mimetype:
|
||||
self.do_stats()
|
||||
raise ValueError(f'Cannot find indexer because {self} does not have a mimetype!')
|
||||
return indexers.find_indexer(self.mimetype)
|
||||
|
||||
def do_index(self):
|
||||
"""Gather any missing information about this file group. Index the contents of this file using an Indexer."""
|
||||
try:
|
||||
if force_index or self.indexed is not True:
|
||||
# Get the indexer on a separate line for debugging.
|
||||
indexer = self.indexer
|
||||
# Only read the contents of the file once.
|
||||
start = now()
|
||||
self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self)
|
||||
if (total_seconds := (now() - start).total_seconds()) > 1:
|
||||
logger.info(f'Indexing {self.path} took {total_seconds} seconds')
|
||||
|
||||
self.indexed = True
|
||||
# Get the indexer on a separate line for debugging.
|
||||
indexer = self.indexer
|
||||
# Only read the contents of the file once.
|
||||
start = now()
|
||||
self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self.primary_path)
|
||||
self.title = self.title or self.primary_path.name
|
||||
if (total_seconds := (now() - start).total_seconds()) > 1:
|
||||
logger.info(f'Indexing {self.primary_path} took {total_seconds} seconds')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index {self.path}', exc_info=e)
|
||||
logger.error(f'Failed to index {self}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
def do_stats(self) -> bool:
|
||||
"""Assign the mimetype, title, size, modification_time of this file.
|
||||
|
||||
Returns True if the file has changed since last index. Change is detected by comparing old size, mimetype
|
||||
modification datetime."""
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, split_file_name_words, get_mimetype
|
||||
|
||||
old_mimetype = self.mimetype
|
||||
old_size = self.size
|
||||
old_modification_datetime = self.modification_datetime
|
||||
|
||||
self.mimetype = get_mimetype(self.path)
|
||||
stem, self.suffix = split_path_stem_and_suffix(self.path)
|
||||
self.full_stem = f'{self.path.parent}/{stem}'
|
||||
self.title = self.title or self.path.name
|
||||
stat = self.path.stat()
|
||||
self.size = stat.st_size
|
||||
self.modification_datetime = from_timestamp(stat.st_mtime)
|
||||
|
||||
changed = old_mimetype != self.mimetype \
|
||||
or old_size != self.size \
|
||||
or old_modification_datetime != self.modification_datetime
|
||||
|
||||
if changed:
|
||||
self.indexed = False
|
||||
# Use file stem as a_text. This may be overwritten by the correct indexer.
|
||||
self.a_text = self.a_text or split_file_name_words(stem)
|
||||
# Clear old indexes, this file has changed and must be re-indexed.
|
||||
self.b_text = self.c_text = self.d_text = None
|
||||
return changed
|
||||
|
||||
@validates('path')
|
||||
def validate_path(self, key, value):
|
||||
value = pathlib.Path(value) if not isinstance(value, pathlib.Path) else value
|
||||
if not value.is_absolute():
|
||||
raise ValueError(f'File path must always be absolute! {value}')
|
||||
|
||||
if not str(value).startswith(str(get_media_directory())):
|
||||
raise ValueError(f'File path {value} not in {get_media_directory()}')
|
||||
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def upsert(cls, path, session: Session) -> Base:
|
||||
if file := session.query(File).filter_by(path=path).one_or_none():
|
||||
return file
|
||||
file = File(path=path)
|
||||
session.add(file)
|
||||
return file
|
||||
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
|
||||
from wrolpi.files.lib import get_primary_file, split_path_stem_and_suffix, get_mimetype
|
||||
file_group = FileGroup()
|
||||
|
||||
file_group.append_files(*paths)
|
||||
file_group.full_stem, _ = split_path_stem_and_suffix(paths[0], full=True)
|
||||
file_group.primary_path = get_primary_file(paths)
|
||||
file_group.mimetype = get_mimetype(file_group.primary_path)
|
||||
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
|
||||
file_group.size = sum(i.stat().st_size for i in paths)
|
||||
|
||||
session.add(file_group)
|
||||
return file_group
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def find_by_path(path, session) -> Optional['FileGroup']:
|
||||
file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
|
||||
return file_group
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
import gc
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import pathlib
|
||||
from typing import List, Dict, Optional
|
||||
from typing import Generator
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import register_modeler, logger
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.common import register_modeler, logger, truncate_generator_bytes, truncate_object_bytes, \
|
||||
split_lines_by_length
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.indexers import Indexer
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import FILE_MAX_PDF_SIZE, PYTEST, FILE_MAX_TEXT_SIZE
|
||||
|
||||
try:
|
||||
from PyPDF2 import PdfReader
|
||||
|
|
@ -14,32 +17,126 @@ except ImportError:
|
|||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['get_pdf_metadata', 'get_words', 'pdf_modeler']
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PDFMetadata:
|
||||
title: str = None
|
||||
author: str = None
|
||||
text: str = None
|
||||
|
||||
|
||||
def create_index(path):
|
||||
file_title = Indexer.get_title(path)
|
||||
|
||||
def get_pdf_title(path: pathlib.Path) -> Optional[str]:
|
||||
reader = PdfReader(path)
|
||||
if reader.metadata and reader.metadata.title:
|
||||
title = reader.metadata.title.strip()
|
||||
# TODO collecting titles causes a memory leak. Why?
|
||||
gc.collect()
|
||||
return title
|
||||
|
||||
data = get_pdf_metadata(reader, path)
|
||||
|
||||
if path.stat().st_size > FILE_MAX_PDF_SIZE:
|
||||
logger.warning(f'PDF too large to fully index: {path}')
|
||||
return data.title, data.author, file_title, None
|
||||
|
||||
words = ''
|
||||
try:
|
||||
# PDFs are complex, don't fail to create title index if text extraction fails.
|
||||
words = '\n'.join(truncate_generator_bytes(get_words(reader, path), FILE_MAX_TEXT_SIZE))
|
||||
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
|
||||
words = split_lines_by_length(words)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index {path}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
return data.title, data.author, file_title, words
|
||||
|
||||
|
||||
def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata:
|
||||
"""Extract title/author from the PDF metadata."""
|
||||
data = PDFMetadata()
|
||||
|
||||
if reader is None or reader.metadata is None:
|
||||
logger.error(f'Cannot get title of {path} PyPDF2 is not installed.')
|
||||
data.title = Indexer.get_title(path)
|
||||
return data
|
||||
|
||||
data.title = reader.metadata.title
|
||||
data.author = reader.metadata.author
|
||||
return data
|
||||
|
||||
|
||||
def get_words(reader: PdfReader, path: pathlib.Path) -> Generator[str, None, None]:
|
||||
"""
|
||||
Reads all text facing up in a PDF.
|
||||
|
||||
Note, may return more than can be stored in Postgres. See: truncate_generator_bytes
|
||||
"""
|
||||
if PdfReader is None:
|
||||
logger.error(f'Cannot index {path} PyPDF2 is not installed.')
|
||||
yield ''
|
||||
return
|
||||
|
||||
for page in reader.pages:
|
||||
# Get all text facing up.
|
||||
text = page.extract_text(0)
|
||||
# Postgres does not allow null characters.
|
||||
text = text.replace('\x00', '\uFFFD').strip()
|
||||
if text:
|
||||
yield text
|
||||
|
||||
|
||||
# PDFs can be large, limit how many are modeled at once.
|
||||
PDF_PROCESSING_LIMIT = 10
|
||||
|
||||
|
||||
@register_modeler
|
||||
def pdf_modeler(groups: Dict[str, List[File]], _: Session):
|
||||
"""Reads the contents of a PDF and extracts the title, attaches the title to the File.
|
||||
|
||||
This modeler does not claim any groups passed to it. Does NOT mark the PDFs as indexed."""
|
||||
async def pdf_modeler():
|
||||
"""Queries for any PDF files that have not been indexed. Each PDF found will be indexed."""
|
||||
if not PdfReader:
|
||||
logger.warning(f'Cannot get title from PDF without PyPDF2')
|
||||
logger.warning(f'Cannot index PDF without PyPDF2')
|
||||
return
|
||||
|
||||
pdf_files = (file for group in groups.values() for file in group if
|
||||
file.indexed is not True and file.mimetype == 'application/pdf')
|
||||
while True:
|
||||
# Continually query for PDFs that have not been indexed.
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup) \
|
||||
.filter(
|
||||
# Get all groups that contain a PDF that have not been indexed.
|
||||
FileGroup.indexed != True,
|
||||
FileGroup.mimetype == 'application/pdf',
|
||||
).limit(PDF_PROCESSING_LIMIT)
|
||||
|
||||
for file in pdf_files:
|
||||
path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
try:
|
||||
if title := get_pdf_title(path):
|
||||
file.title = title
|
||||
except Exception as e:
|
||||
logger.error(f'Unable to get title of {file.path}', exc_info=e)
|
||||
processed = 0
|
||||
for file_group in file_groups:
|
||||
processed += 1
|
||||
pdf_files = file_group.my_files('application/pdf')
|
||||
if not pdf_files:
|
||||
logger.error('Query returned a group without a PDF!')
|
||||
continue
|
||||
|
||||
# Assume the first PDF is the only PDF.
|
||||
pdf_file = pdf_files[0]['path']
|
||||
try:
|
||||
title, author, file_title, contents = create_index(pdf_file)
|
||||
file_group.data = {'author': author}
|
||||
file_group.title = file_group.a_text = title
|
||||
file_group.b_text = author
|
||||
file_group.c_text = file_title
|
||||
file_group.d_text = contents
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index PDF {pdf_file}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
|
||||
file_group.indexed = True
|
||||
|
||||
if processed:
|
||||
logger.debug(f'pdf_modeler processed {processed} PDFs')
|
||||
|
||||
if processed < PDF_PROCESSING_LIMIT:
|
||||
# Did not reach limit, do not query again.
|
||||
break
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
|
|
|||
|
|
@ -19,17 +19,12 @@ class FilesSearchRequest:
|
|||
offset: Optional[int] = 0
|
||||
mimetypes: List[str] = field(default_factory=list)
|
||||
model: Optional[str] = None
|
||||
tag_names: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DirectoryRefreshRequest:
|
||||
directory: str = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class FilesRefreshListRequest:
|
||||
files: List[str] = None
|
||||
include_files_near: bool = True
|
||||
class FilesRefreshRequest:
|
||||
paths: Optional[List[str]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -40,3 +35,9 @@ class DirectoriesRequest:
|
|||
@dataclass
|
||||
class DirectoriesResponse:
|
||||
directories: List[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TagFileGroupPost:
|
||||
file_group_id: int
|
||||
tag_name: str
|
||||
|
|
|
|||
|
|
@ -1,15 +1,13 @@
|
|||
import json
|
||||
from http import HTTPStatus
|
||||
from itertools import zip_longest
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.errors import API_ERRORS, WROLModeEnabled
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
def test_list_files_api(test_client, make_files_structure, test_directory):
|
||||
|
|
@ -21,7 +19,7 @@ def test_list_files_api(test_client, make_files_structure, test_directory):
|
|||
'empty directory/',
|
||||
'videos/other video.mp4',
|
||||
'videos/some video.mp4',
|
||||
'lost+found/', # Should always be ignored.
|
||||
'lost+found/', # Should always be ignored.
|
||||
]
|
||||
files = make_files_structure(files)
|
||||
files[0].write_text('bar contents')
|
||||
|
|
@ -154,19 +152,9 @@ def test_delete_wrol_mode(test_client, wrol_mode_fixture):
|
|||
assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
|
||||
|
||||
def do_search(test_client, search_str, total, expected):
|
||||
content = json.dumps({'search_str': search_str})
|
||||
request, response = test_client.post('/api/files/search', content=content)
|
||||
assert response.json['totals']['files'] == total
|
||||
for file, expected in zip_longest(response.json['files'], expected):
|
||||
assert_dict_contains(file, expected)
|
||||
# FileBrowser in React requires a key. We use the path.
|
||||
assert file['path'] == file['key']
|
||||
|
||||
|
||||
def test_files_search(test_session, test_client, make_files_structure):
|
||||
def test_files_search(test_session, test_client, make_files_structure, assert_files_search):
|
||||
# You can search an empty directory.
|
||||
do_search(test_client, 'nothing', 0, [])
|
||||
assert_files_search('nothing', [])
|
||||
|
||||
# Create files in the temporary directory. Add some contents so the mimetype can be tested.
|
||||
files = [
|
||||
|
|
@ -185,65 +173,11 @@ def test_files_search(test_session, test_client, make_files_structure):
|
|||
request, response = test_client.post('/api/files/refresh')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)])
|
||||
do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)])
|
||||
do_search(test_client, 'baz', 2, [
|
||||
dict(path='baz baz two.mp4', mimetype='video/mp4', size=1056318),
|
||||
dict(path='baz.mp4', mimetype='video/mp4', size=1056318),
|
||||
])
|
||||
do_search(test_client, 'two', 1, [dict(path='baz baz two.mp4', mimetype='video/mp4', size=1056318)])
|
||||
do_search(test_client, 'nothing', 0, [])
|
||||
|
||||
# No search string returns all Files.
|
||||
do_search(test_client, None, 4, [
|
||||
dict(path='archives/bar.txt'),
|
||||
dict(path='baz baz two.mp4'),
|
||||
dict(path='baz.mp4'),
|
||||
dict(path='foo_is_the_name.txt'),
|
||||
])
|
||||
|
||||
|
||||
def test_associated_files(test_session, test_client, make_files_structure):
|
||||
mp4, png, j, txt = make_files_structure([
|
||||
'video.mp4',
|
||||
'video.png',
|
||||
'video.info.json',
|
||||
'not a video.txt',
|
||||
])
|
||||
test_client.post('/api/files/refresh')
|
||||
mp4_file = test_session.query(File).filter_by(path=mp4).one()
|
||||
png_file = test_session.query(File).filter_by(path=png).one()
|
||||
j_file = test_session.query(File).filter_by(path=j).one()
|
||||
txt_file = test_session.query(File).filter_by(path=txt).one()
|
||||
|
||||
def check_files_search(content, expected):
|
||||
request, response = test_client.post('/api/files/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert [{'path': i['path']} for i in response.json['files']] == expected
|
||||
|
||||
check_files_search(
|
||||
{'search_str': 'video'},
|
||||
[
|
||||
{'path': 'not a video.txt'},
|
||||
{'path': 'video.info.json'},
|
||||
{'path': 'video.mp4'},
|
||||
{'path': 'video.png'},
|
||||
]
|
||||
)
|
||||
|
||||
mp4_file.associated = False # the modeled file
|
||||
png_file.associated = True
|
||||
j_file.associated = True
|
||||
test_session.commit()
|
||||
|
||||
# Associated files are last.
|
||||
check_files_search(
|
||||
{'search_str': 'video'},
|
||||
[
|
||||
{'path': 'not a video.txt'},
|
||||
{'path': 'video.mp4'},
|
||||
]
|
||||
)
|
||||
assert_files_search('foo', [dict(full_stem='foo_is_the_name')])
|
||||
assert_files_search('bar', [dict(full_stem='archives/bar')])
|
||||
assert_files_search('baz', [dict(full_stem='baz baz two'), dict(full_stem='baz')])
|
||||
assert_files_search('two', [dict(full_stem='baz baz two')])
|
||||
assert_files_search('nothing', [])
|
||||
|
||||
|
||||
def test_directory_search(test_client, make_files_structure):
|
||||
|
|
@ -271,19 +205,33 @@ def test_directory_search(test_client, make_files_structure):
|
|||
assert_directories('food', [])
|
||||
|
||||
|
||||
def test_refresh_files_list(test_session, test_client, make_files_structure):
|
||||
def test_refresh_files_list(test_session, test_client, make_files_structure, test_directory):
|
||||
"""The user can request to refresh specific files."""
|
||||
make_files_structure(['bar.txt', 'bar.mp4'])
|
||||
|
||||
# Video file near `bar.txt` can be ignored.
|
||||
content = json.dumps({'files': ['bar.txt'], 'include_files_near': False})
|
||||
request, response = test_client.post('/api/files/refresh/list', content=content)
|
||||
# Only the single file that was refreshed is discovered.
|
||||
content = json.dumps({'paths': ['bar.txt']})
|
||||
request, response = test_client.post('/api/files/refresh', content=content)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert test_session.query(File).count() == 1
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
group: FileGroup = test_session.query(FileGroup).one()
|
||||
assert len(group.files) == 1
|
||||
|
||||
request, response = test_client.post('/api/files/refresh')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
group: FileGroup = test_session.query(FileGroup).one()
|
||||
assert len(group.files) == 2
|
||||
|
||||
|
||||
def test_file_statistics(test_session, test_client, example_pdf, example_mobi, example_epub, video_file):
|
||||
def test_file_statistics(test_session, test_client, test_directory, example_pdf, example_mobi, example_epub,
|
||||
video_file):
|
||||
"""A summary of File statistics can be fetched."""
|
||||
# Give each file a unique stem.
|
||||
video_file.rename(test_directory / 'video.mp4')
|
||||
example_pdf.rename(test_directory / 'pdf.pdf')
|
||||
example_mobi.rename(test_directory / 'mobi.mobi')
|
||||
example_epub.rename(test_directory / 'epub.epub')
|
||||
|
||||
# Statistics can be fetched while empty.
|
||||
request, response = test_client.get('/api/statistics')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
|
@ -311,7 +259,7 @@ def test_file_statistics(test_session, test_client, example_pdf, example_mobi, e
|
|||
'ebook_count': 2,
|
||||
'image_count': 0,
|
||||
'pdf_count': 1,
|
||||
'total_count': 5,
|
||||
'total_count': 4,
|
||||
'video_count': 1,
|
||||
'zip_count': 0,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import json
|
||||
import shutil
|
||||
from http import HTTPStatus
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.ebooks import EBook, MOBI_MIMETYPE, EPUB_MIMETYPE
|
||||
from wrolpi.files.ebooks import EBook, EPUB_MIMETYPE
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
|
|
@ -17,38 +17,53 @@ async def test_index(test_session, test_directory, example_epub, example_mobi):
|
|||
Covers can be discovered."""
|
||||
await refresh_files()
|
||||
|
||||
ebook_epub, ebook_mobi = test_session.query(EBook).order_by(EBook.ebook_path)
|
||||
ebook_epub: EBook
|
||||
ebook_mobi: EBook
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
|
||||
assert ebook_epub.title == 'WROLPi Test Book'
|
||||
assert ebook_epub.ebook_file.mimetype.startswith(EPUB_MIMETYPE)
|
||||
assert ebook_epub.creator == 'roland'
|
||||
assert ebook_epub.size == ebook_epub.ebook_file.size
|
||||
assert ebook_epub.cover_path and ebook_epub.cover_path.is_file()
|
||||
# The cover is the WROLPi logo.
|
||||
assert ebook_epub.cover_file.size == 297099
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.file_group.mimetype.startswith(EPUB_MIMETYPE)
|
||||
assert ebook.creator == 'roland'
|
||||
assert ebook.size == 292579
|
||||
assert ebook.file_group.data == {'creator': 'roland', 'title': 'WROLPi Test Book',
|
||||
'cover_path': test_directory / 'example.jpeg',
|
||||
'ebook_path': test_directory / 'example.epub',
|
||||
}
|
||||
|
||||
assert ebook_epub.ebook_file.a_text, 'Book title was not indexed'
|
||||
assert ebook_epub.ebook_file.b_text, 'Book creator was not indexed'
|
||||
assert ebook_epub.ebook_file.d_text, 'Book text was not indexed'
|
||||
|
||||
# Mobi is not fully supported, title is the file name. No creator or cover.
|
||||
assert ebook_mobi.title == 'example'
|
||||
assert ebook_mobi.ebook_file.mimetype == MOBI_MIMETYPE
|
||||
assert not ebook_mobi.creator
|
||||
assert not ebook_mobi.cover_path
|
||||
assert ebook.file_group.a_text, 'Book title was not indexed'
|
||||
assert ebook.file_group.b_text, 'Book creator was not indexed'
|
||||
assert ebook.file_group.d_text, 'Book text was not indexed'
|
||||
# Both ebooks are assumed to be the same book, but different formats.
|
||||
assert (epubs := ebook.file_group.my_files('application/epub+zip')) and len(epubs) == 1 \
|
||||
and epubs[0]['path'] == example_epub
|
||||
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
|
||||
and mobis[0]['path'] == example_mobi
|
||||
# Cover was discovered.
|
||||
assert len(ebook.file_group.my_poster_files()) == 1
|
||||
# EPUB, MOBI, JPEG.
|
||||
assert len(ebook.file_group.files) == 3
|
||||
|
||||
# Ebooks can be deleted during refresh.
|
||||
example_mobi.unlink()
|
||||
example_epub.unlink()
|
||||
await refresh_files()
|
||||
assert test_session.query(EBook).count() == 0
|
||||
assert test_session.query(EBook).count() == 1
|
||||
assert not ebook.file_group.my_files('application/epub+zip')
|
||||
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
|
||||
and mobis[0]['path'] == example_mobi
|
||||
# Cover was discovered.
|
||||
assert len(ebook.file_group.my_poster_files()) == 1
|
||||
# MOBI, JPEG.
|
||||
assert len(ebook.file_group.files) == 2
|
||||
|
||||
# Extract/index should only be done once.
|
||||
with mock.patch('wrolpi.files.ebooks.extract_ebook_data') as mock_extract_ebook_data:
|
||||
mock_extract_ebook_data.side_effect = Exception('extract_ebook_data should not be called again')
|
||||
await refresh_files()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory):
|
||||
cover_path = example_epub.with_suffix('.jpg')
|
||||
cover_path.write_bytes(image_bytes_factory())
|
||||
await refresh_files()
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
|
||||
# Cover file near the eBook was discovered.
|
||||
assert ebook.cover_path.read_bytes() == cover_path.read_bytes()
|
||||
|
||||
|
||||
def test_search(test_session, test_client, example_epub):
|
||||
|
|
@ -59,75 +74,59 @@ def test_search(test_session, test_client, example_epub):
|
|||
assert test_session.query(EBook).count() == 1
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.creator == 'roland'
|
||||
assert ebook.ebook_file.a_text == 'WROLPi Test Book', 'Book title was not updated'
|
||||
assert ebook.ebook_file.d_text, 'Book was not indexed'
|
||||
assert ebook.file_group.a_text == 'WROLPi Test Book', 'Book title was not updated'
|
||||
assert ebook.file_group.d_text, 'Book was not indexed'
|
||||
|
||||
content = dict(mimetypes=['application/epub', 'application/x-mobipocket-ebook'])
|
||||
request, response = test_client.post('/api/files/search', content=json.dumps(content))
|
||||
assert response.status == HTTPStatus.OK
|
||||
assert response.json
|
||||
result = response.json['files'][0]
|
||||
assert result['path'] == 'example.epub'
|
||||
assert result['mimetype'] == 'application/epub+zip'
|
||||
file_group = response.json['file_groups'][0]
|
||||
epub_file = file_group['files'][0]
|
||||
assert epub_file['path'] == 'example.epub' and epub_file['mimetype'] == 'application/epub+zip'
|
||||
assert_dict_contains(
|
||||
result['ebook'],
|
||||
{'cover_path': 'example.jpeg', 'ebook_path': 'example.epub'},
|
||||
file_group['data'],
|
||||
{'cover_path': 'example.jpeg', 'ebook_path': 'example.epub', 'title': 'WROLPi Test Book', 'creator': 'roland'},
|
||||
)
|
||||
|
||||
content = dict(mimetypes=['application/epub+zip', 'application/x-mobipocket-ebook'])
|
||||
request, response = test_client.post('/api/files/search', content=json.dumps(content))
|
||||
result = response.json['files'][0]
|
||||
assert result['path'] == 'example.epub'
|
||||
assert result['mimetype'] == 'application/epub+zip'
|
||||
|
||||
content = dict(mimetypes=[])
|
||||
# No Mobi ebook.
|
||||
content = dict(mimetypes=['application/x-mobipocket-ebook'])
|
||||
request, response = test_client.post('/api/files/search', content=json.dumps(content))
|
||||
assert response.status == HTTPStatus.OK
|
||||
assert response.json
|
||||
assert len(response.json['file_groups']) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_calibre_cover(test_session, test_directory, example_epub, image_file):
|
||||
async def test_discover_calibre_cover(test_session, test_directory, example_epub, example_mobi, image_file):
|
||||
"""Calibre puts a cover near an ebook file, test if it can be found."""
|
||||
# Create two files in the
|
||||
# Create a Calibre metadata file.
|
||||
metadata = test_directory / 'metadata.opf'
|
||||
metadata.touch()
|
||||
# Create an alternate format next to the epub. This will be ignored because it is not yet supported.
|
||||
mobi_path = example_epub.with_suffix('.mobi')
|
||||
mobi_path.touch()
|
||||
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(EBook).count() == 1
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
# The cover in the ebook was extracted.
|
||||
assert ebook.cover_file.size == 297099
|
||||
assert ebook.cover_file['path'] == test_directory / 'example.jpeg'
|
||||
assert (test_directory / 'example.jpeg').is_file()
|
||||
|
||||
# Delete the extracted cover, use the cover image from Calibre.
|
||||
ebook.cover_path.unlink()
|
||||
test_session.delete(ebook.cover_file)
|
||||
(test_directory / 'example.jpeg').unlink()
|
||||
cover_image = test_directory / 'cover.jpg'
|
||||
shutil.move(image_file, cover_image)
|
||||
|
||||
# Reset files.
|
||||
for file_group in test_session.query(FileGroup):
|
||||
test_session.delete(file_group)
|
||||
test_session.commit()
|
||||
|
||||
# Calibre cover image was discovered, no cover was generated.
|
||||
await refresh_files()
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file.size == 641
|
||||
|
||||
# Create a file nearby, its no longer a Calibre directory.
|
||||
mobi_path = mobi_path.rename(test_directory / 'other book.mobi')
|
||||
await refresh_files()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file.size == 297099
|
||||
mobi_path.unlink()
|
||||
|
||||
# Delete the discovered cover, cover should be generated.
|
||||
cover_image.unlink()
|
||||
await refresh_files()
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file.size == 297099
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file['path'] == test_directory / 'cover.jpg'
|
||||
|
|
|
|||
|
|
@ -10,21 +10,17 @@ from uuid import uuid4
|
|||
import mock
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules import videos
|
||||
from wrolpi.common import get_media_directory, timer
|
||||
from wrolpi.errors import InvalidFile
|
||||
from wrolpi.common import timer
|
||||
from wrolpi.dates import now, from_timestamp
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory
|
||||
from wrolpi.files import lib, indexers
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import TagFile
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
def assert_files(session: Session, expected):
|
||||
files = {str(i.path.relative_to(get_media_directory())) for i in session.query(File).all()}
|
||||
assert files == set(expected)
|
||||
|
||||
|
||||
def test_delete_file(make_files_structure, test_directory):
|
||||
"""
|
||||
File in the media directory can be deleted.
|
||||
|
|
@ -64,6 +60,8 @@ def test_delete_file(make_files_structure, test_directory):
|
|||
('/absolute/foo-something.info.json', ('foo-something', '.info.json')),
|
||||
('/absolute/foo', ('foo', '')),
|
||||
('/absolute/foo.bar', ('foo', '.bar')),
|
||||
('foo.en.srt', ('foo', '.en.srt')),
|
||||
('foo.pl.srt', ('foo', '.pl.srt')),
|
||||
]
|
||||
)
|
||||
def test_split_path_stem_and_suffix(path, expected):
|
||||
|
|
@ -71,7 +69,7 @@ def test_split_path_stem_and_suffix(path, expected):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_files(test_session, make_files_structure, test_directory):
|
||||
async def test_refresh_files(test_session, make_files_structure, assert_file_groups):
|
||||
"""All files in the media directory should be found when calling `refresh_files`"""
|
||||
files = make_files_structure([
|
||||
'foo.txt',
|
||||
|
|
@ -81,18 +79,188 @@ async def test_refresh_files(test_session, make_files_structure, test_directory)
|
|||
foo, bar, baz = files
|
||||
|
||||
await lib.refresh_files()
|
||||
assert_files(test_session, ['bar.txt', 'baz.txt', 'foo.txt'])
|
||||
assert_file_groups([
|
||||
{'full_stem': 'foo', 'indexed': True},
|
||||
{'full_stem': 'bar', 'indexed': True},
|
||||
{'full_stem': 'baz', 'indexed': True}])
|
||||
|
||||
baz.unlink()
|
||||
|
||||
await lib.refresh_files()
|
||||
assert_files(test_session, ['bar.txt', 'foo.txt'])
|
||||
assert_file_groups([{'full_stem': 'foo', 'indexed': True}, {'full_stem': 'bar', 'indexed': True}])
|
||||
|
||||
foo.unlink()
|
||||
bar.unlink()
|
||||
|
||||
await lib.refresh_files()
|
||||
assert_files(test_session, [])
|
||||
assert_file_groups([{'full_stem': 'bar', 'indexed': True}])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_bogus_files(test_session, make_files_structure, test_directory, assert_file_groups,
|
||||
insert_file_group):
|
||||
"""Bogus files are removed during a refresh."""
|
||||
make_files_structure(['does exist.txt'])
|
||||
await lib.refresh_files()
|
||||
insert_file_group([test_directory / 'does not exist.txt'])
|
||||
test_session.commit()
|
||||
|
||||
# Bogus file was inserted.
|
||||
assert_file_groups([
|
||||
{'full_stem': 'does exist', 'indexed': True},
|
||||
{'full_stem': 'does not exist', 'indexed': True},
|
||||
])
|
||||
|
||||
await lib.refresh_files()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 1, 'Bogus file was not removed'
|
||||
assert_file_groups([{'full_stem': 'does exist', 'indexed': True}])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_empty_media_directory(test_session, test_directory):
|
||||
"""refresh_paths will refuse to refresh with an empty media directory."""
|
||||
with pytest.raises(UnknownDirectory):
|
||||
await lib.refresh_files()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test__upsert_files(test_session, make_files_structure, test_directory, assert_file_groups, video_file,
|
||||
srt_file3):
|
||||
bar, baz = make_files_structure({
|
||||
'dir1/bar.txt': None,
|
||||
'baz.txt': 'baz file',
|
||||
})
|
||||
video_file = video_file.rename(test_directory / 'video.mp4')
|
||||
srt_file3 = srt_file3.rename(test_directory / 'video.en.srt')
|
||||
video_stem = str(test_directory / video_file.stem)
|
||||
foo_mtime = from_timestamp(srt_file3.stat().st_mtime)
|
||||
|
||||
# All files are found because they are in this refresh request, or in the `dir1` directory.
|
||||
idempotency = now()
|
||||
lib._upsert_files([video_file, srt_file3, bar, baz], idempotency)
|
||||
assert_file_groups([
|
||||
{'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': False,
|
||||
'files': [
|
||||
{'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
|
||||
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
|
||||
]},
|
||||
{'primary_path': bar, 'idempotency': idempotency,
|
||||
'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}]
|
||||
},
|
||||
{'primary_path': baz, 'idempotency': idempotency,
|
||||
'files': [{'path': baz, 'size': 8, 'suffix': '.txt', 'mimetype': 'text/plain'}]
|
||||
},
|
||||
])
|
||||
|
||||
# Modified files should be re-indexed.
|
||||
bar.touch()
|
||||
baz.write_text('new baz')
|
||||
# Simulate that the video was indexed, it should not be re-indexed because it was not modified.
|
||||
test_session.query(FileGroup).filter_by(primary_path=video_file).one().indexed = True
|
||||
test_session.commit()
|
||||
assert_file_groups(
|
||||
[{'full_stem': video_stem, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True}],
|
||||
assert_count=False)
|
||||
|
||||
# Only modified files need to be re-indexed.
|
||||
lib._upsert_files([video_file, srt_file3, bar, baz], idempotency)
|
||||
assert_file_groups([
|
||||
{'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True,
|
||||
'files': [
|
||||
{'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
|
||||
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
|
||||
]},
|
||||
{'primary_path': bar, 'idempotency': idempotency,
|
||||
'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}],
|
||||
},
|
||||
{'primary_path': baz, 'idempotency': idempotency,
|
||||
'files': [{'path': baz, 'size': 7, 'suffix': '.txt', 'mimetype': 'text/plain'}],
|
||||
},
|
||||
])
|
||||
|
||||
# Deleting SRT removes it from the video.
|
||||
srt_file3.unlink()
|
||||
lib._upsert_files([video_file, bar, baz], idempotency)
|
||||
video_file_group: FileGroup = test_session.query(FileGroup).filter_by(full_stem=str(test_directory / 'video')).one()
|
||||
assert len(video_file_group.files) == 1, 'SRT file was not removed from files'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_discover_paths(test_session, make_files_structure, test_directory, assert_files,
|
||||
assert_file_groups):
|
||||
foo, bar, baz = make_files_structure(['dir1/foo.txt', 'dir1/bar.txt', 'baz.txt'])
|
||||
dir1 = foo.parent
|
||||
|
||||
# `refresh_paths` only refreshes the file requested.
|
||||
await lib.refresh_discover_paths([foo, ])
|
||||
assert_files([
|
||||
{'path': 'dir1/foo.txt'},
|
||||
])
|
||||
|
||||
# `refresh_paths` refreshes recursively.
|
||||
await lib.refresh_discover_paths([foo.parent, ])
|
||||
assert_files([
|
||||
{'path': 'dir1/foo.txt'},
|
||||
{'path': 'dir1/bar.txt'},
|
||||
])
|
||||
|
||||
# `refresh_paths` finally discovers the file at the top of the media directory.
|
||||
await lib.refresh_discover_paths([test_directory, ])
|
||||
assert_files([
|
||||
{'path': 'dir1/foo.txt'},
|
||||
{'path': 'dir1/bar.txt'},
|
||||
{'path': 'baz.txt'},
|
||||
])
|
||||
|
||||
# Records for deleted files are deleted. Request a refresh of `dir1` so we indirectly refresh `foo`.
|
||||
foo.unlink()
|
||||
await lib.refresh_discover_paths([dir1, ])
|
||||
assert_files([
|
||||
{'path': 'dir1/bar.txt'},
|
||||
{'path': 'baz.txt'},
|
||||
])
|
||||
|
||||
# Records for all children of a directory are deleted.
|
||||
bar.unlink()
|
||||
dir1.rmdir()
|
||||
await lib.refresh_discover_paths([dir1, ])
|
||||
assert_files([
|
||||
{'path': 'baz.txt'},
|
||||
])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_discover_paths_groups(test_session, make_files_structure, test_directory):
|
||||
make_files_structure(['dir1/foo.mp4', 'dir1/foo.info.json', 'baz.txt'])
|
||||
await lib.refresh_discover_paths([test_directory, ], now())
|
||||
|
||||
# Two "foo" files, one "baz" file.
|
||||
assert test_session.query(FileGroup).count() == 2
|
||||
|
||||
baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
||||
# "foo" files are related to the "foo" group.
|
||||
assert sorted([str(i['path'].relative_to(test_directory)) for i in foo.my_files()]) == [
|
||||
'dir1/foo.info.json', 'dir1/foo.mp4',
|
||||
]
|
||||
# "bar" file is the only file related to the "bar" group.
|
||||
assert len(baz.my_files()) == 1
|
||||
assert str(baz.my_files()[0]['path'].relative_to(test_directory)) == 'baz.txt'
|
||||
assert baz.indexed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_tag(test_session, make_files_structure, test_directory, tag_factory):
|
||||
"""A FileGroup can be tagged."""
|
||||
make_files_structure(['foo.mp4'])
|
||||
await lib.refresh_discover_paths([test_directory, ], now())
|
||||
one = tag_factory()
|
||||
|
||||
foo: FileGroup = test_session.query(FileGroup).one()
|
||||
foo.add_tag(one)
|
||||
test_session.commit()
|
||||
|
||||
tag_file2: TagFile = test_session.query(TagFile).one()
|
||||
assert tag_file2 and tag_file2.file_group == foo
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -102,8 +270,8 @@ async def test_refresh_a_text_no_indexer(test_session, make_files_structure):
|
|||
|
||||
await lib.refresh_files()
|
||||
|
||||
files = {i.a_text for i in test_session.query(File).all()}
|
||||
assert files == {'{bar,bar-bar}', '{foo}'}
|
||||
files = {i.a_text for i in test_session.query(FileGroup)}
|
||||
assert files == {'bar bar bar-bar', 'foo'}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -113,51 +281,11 @@ async def test_refresh_many_files(test_session, make_files_structure):
|
|||
make_files_structure([f'{uuid4()}.txt' for _ in range(count)])
|
||||
with timer('first refresh'):
|
||||
await lib.refresh_files()
|
||||
assert test_session.query(File).count() == count
|
||||
assert test_session.query(FileGroup).count() == count
|
||||
|
||||
with timer('second refresh'):
|
||||
await lib.refresh_files()
|
||||
assert test_session.query(File).count() == count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_files_list(test_session, make_files_structure, test_directory):
|
||||
make_files_structure(['foo.txt', 'bar.txt'])
|
||||
|
||||
# Only foo.txt should have been refreshed.
|
||||
await lib.refresh_files_list(['foo.txt'])
|
||||
assert_files(test_session, ['foo.txt'])
|
||||
|
||||
# Both files should be found
|
||||
await lib.refresh_files_list(['bar.txt'])
|
||||
assert_files(test_session, ['foo.txt', 'bar.txt'])
|
||||
|
||||
# Both files should be unchanged.
|
||||
await lib.refresh_files_list(['foo.txt', 'bar.txt'])
|
||||
assert_files(test_session, ['foo.txt', 'bar.txt'])
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
# Some files must be refreshed.
|
||||
await lib.refresh_files_list([])
|
||||
|
||||
# Files that share the stem can also be refreshed.
|
||||
(test_directory / 'foo.mp4').touch()
|
||||
await lib.refresh_files_list(['foo.txt', 'bar.txt'])
|
||||
assert_files(test_session, ['foo.txt', 'foo.mp4', 'bar.txt'])
|
||||
|
||||
# Files that share the stem can also be ignored.
|
||||
(test_directory / 'bar.mp4').touch()
|
||||
await lib.refresh_files_list(['bar.txt'], include_files_near=False)
|
||||
assert_files(test_session, ['foo.txt', 'foo.mp4', 'bar.txt'])
|
||||
|
||||
# Deleted files are removed.
|
||||
(test_directory / 'foo.mp4').unlink()
|
||||
(test_directory / 'bar.txt').unlink()
|
||||
await lib.refresh_files_list(['foo.txt', 'bar.txt'], include_files_near=False)
|
||||
assert_files(test_session, ['foo.txt', ])
|
||||
# bar.mp4 is discovered near non-existent bar.txt.
|
||||
await lib.refresh_files_list(['foo.txt', 'bar.txt'])
|
||||
assert_files(test_session, ['foo.txt', 'bar.mp4'])
|
||||
assert test_session.query(FileGroup).count() == count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -181,33 +309,9 @@ async def test_refresh_cancel(test_session, make_files_structure, test_directory
|
|||
task = asyncio.create_task(lib.refresh_files())
|
||||
await assert_cancel(task)
|
||||
|
||||
task = asyncio.create_task(lib.refresh_directory_files_recursively(test_directory))
|
||||
await assert_cancel(task)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_files_in_directory(test_session, make_files_structure, test_directory):
|
||||
"""A subdirectory can be refreshed, files above it can be ignored."""
|
||||
ignored, foo, similar = make_files_structure([
|
||||
'ignored.txt',
|
||||
'subdir/foo.txt',
|
||||
'subdir-similarly-named.mp4',
|
||||
])
|
||||
|
||||
await lib.refresh_directory_files_recursively(test_directory / 'subdir')
|
||||
assert_files(test_session, ['subdir/foo.txt'])
|
||||
|
||||
await lib.refresh_directory_files_recursively(test_directory)
|
||||
assert_files(test_session, ['ignored.txt', 'subdir/foo.txt', 'subdir-similarly-named.mp4'])
|
||||
|
||||
# The similarly named file is not deleted when refreshing the directory which shares the name.
|
||||
foo.unlink()
|
||||
await lib.refresh_directory_files_recursively(test_directory / 'subdir')
|
||||
assert_files(test_session, ['ignored.txt', 'subdir-similarly-named.mp4'])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mime_type(test_session, make_files_structure, test_directory):
|
||||
async def test_mime_type(test_session, make_files_structure, test_directory, assert_files):
|
||||
"""Files module uses the `file` command to get the mimetype of each file."""
|
||||
from PIL import Image
|
||||
|
||||
|
|
@ -222,12 +326,17 @@ async def test_mime_type(test_session, make_files_structure, test_directory):
|
|||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', baz)
|
||||
|
||||
await lib.refresh_files()
|
||||
assert_files(test_session, ['dir/foo text.txt', 'dir/bar.jpeg', 'dir/baz.mp4', 'dir/empty'])
|
||||
assert_files([
|
||||
{'path': 'dir/foo text.txt'},
|
||||
{'path': 'dir/bar.jpeg'},
|
||||
{'path': 'dir/baz.mp4'},
|
||||
{'path': 'dir/empty'},
|
||||
])
|
||||
|
||||
foo = test_session.query(File).filter_by(path=f'{test_directory}/dir/foo text.txt').one()
|
||||
bar = test_session.query(File).filter_by(path=f'{test_directory}/dir/bar.jpeg').one()
|
||||
baz = test_session.query(File).filter_by(path=f'{test_directory}/dir/baz.mp4').one()
|
||||
empty = test_session.query(File).filter_by(path=f'{test_directory}/dir/empty').one()
|
||||
foo = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/foo text.txt').one()
|
||||
bar = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/bar.jpeg').one()
|
||||
baz = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/baz.mp4').one()
|
||||
empty = test_session.query(FileGroup).filter_by(primary_path=f'{test_directory}/dir/empty').one()
|
||||
|
||||
assert foo.mimetype == 'text/plain'
|
||||
assert bar.mimetype == 'image/jpeg'
|
||||
|
|
@ -257,29 +366,19 @@ async def test_files_indexer(test_session, make_files_structure, test_directory)
|
|||
|
||||
# Enable slow feature for testing.
|
||||
# TODO can this be sped up to always be included?
|
||||
with mock.patch('modules.videos.EXTRACT_SUBTITLES', True):
|
||||
with mock.patch('modules.videos.lib.EXTRACT_SUBTITLES', True):
|
||||
await lib.refresh_files()
|
||||
|
||||
text_file, zip_file, image_file, unknown_file, info_json_file, video_file \
|
||||
= test_session.query(File).order_by(File.path)
|
||||
text_file, zip_file, image_file, unknown_file, video_file \
|
||||
= test_session.query(FileGroup).order_by(FileGroup.full_stem)
|
||||
|
||||
# Indexers are detected correctly.
|
||||
assert text_file.path.suffix == '.txt' and text_file.indexer == indexers.TextIndexer
|
||||
assert zip_file.path.suffix == '.zip' and zip_file.indexer == indexers.ZipIndexer
|
||||
assert image_file.path.suffix == '.jpeg' and image_file.indexer == indexers.DefaultIndexer
|
||||
assert unknown_file.path.suffix == '' and unknown_file.indexer == indexers.DefaultIndexer
|
||||
assert info_json_file.path.suffix == '.json' and info_json_file.indexer == indexers.DefaultIndexer
|
||||
assert video_file.path.suffix == '.mp4' and video_file.indexer == videos.VideoIndexer
|
||||
|
||||
def assert_file_properties(file: File, suffix):
|
||||
assert file.suffix == suffix
|
||||
|
||||
assert_file_properties(text_file, '.txt')
|
||||
assert_file_properties(zip_file, '.zip')
|
||||
assert_file_properties(image_file, '.jpeg')
|
||||
assert_file_properties(unknown_file, '')
|
||||
assert_file_properties(info_json_file, '.info.json')
|
||||
assert_file_properties(video_file, '.mp4')
|
||||
assert text_file.mimetype == 'text/plain' and text_file.indexer == indexers.TextIndexer
|
||||
assert zip_file.mimetype == 'application/zip' and zip_file.indexer == indexers.ZipIndexer
|
||||
assert image_file.mimetype == 'image/jpeg' and image_file.indexer == indexers.DefaultIndexer
|
||||
assert unknown_file.mimetype == 'inode/x-empty' and unknown_file.indexer == indexers.DefaultIndexer
|
||||
# Video are indexed by the modeler, not by an indexer.
|
||||
assert video_file.mimetype == 'video/mp4' and video_file.indexer == indexers.DefaultIndexer
|
||||
|
||||
# File are indexed by their titles and contents.
|
||||
files, total = lib.search_files('file', 10, 0)
|
||||
|
|
@ -289,15 +388,15 @@ async def test_files_indexer(test_session, make_files_structure, test_directory)
|
|||
files, total = lib.search_files('contents', 10, 0)
|
||||
assert total == 1 and files[0]['title'] == 'a text file.txt', 'The text file contains "contents".'
|
||||
files, total = lib.search_files('video', 10, 0)
|
||||
assert total == 1 and {i['title'] for i in files} == {'a video file.mp4'}, 'The video file contains "video".'
|
||||
assert total == 1 and {i['title'] for i in files} == {'a video file'}, 'The video file contains "video".'
|
||||
files, total = lib.search_files('yawn', 10, 0)
|
||||
assert total == 1 and files[0]['title'] == 'a video file.mp4', 'The video file captions contain "yawn".'
|
||||
assert total == 1 and files[0]['title'] == 'a video file', 'The video file captions contain "yawn".'
|
||||
files, total = lib.search_files('bunny', 10, 0)
|
||||
assert total == 1 and {i['title'] for i in files} == {'a zip file.zip'}, \
|
||||
'The zip file contains a file with "bunny" in the title.'
|
||||
|
||||
with mock.patch('modules.videos.VideoIndexer.create_index') as mock_create_index:
|
||||
mock_create_index.side_effect = Exception('This should not be called twice')
|
||||
with mock.patch('modules.videos.models.Video.validate') as mock_validate:
|
||||
mock_validate.side_effect = Exception('This should not be called twice')
|
||||
await lib.refresh_files()
|
||||
|
||||
# Change the contents, the file should be re-indexed.
|
||||
|
|
@ -308,11 +407,11 @@ async def test_files_indexer(test_session, make_files_structure, test_directory)
|
|||
|
||||
|
||||
@pytest.mark.parametrize('name,expected', [
|
||||
('this.txt', ['this', 'txt']),
|
||||
('name', ['name']),
|
||||
('name two', ['name', 'two']),
|
||||
('this self-reliance_split.txt', ['this', 'self', 'reliance', 'self-reliance', 'split', 'txt']),
|
||||
('-be_split!.txt', ['-be', 'split!', 'txt']),
|
||||
('this.txt', 'this txt'),
|
||||
('name', 'name'),
|
||||
('name two', 'name two'),
|
||||
('this self-reliance_split.txt', 'this self reliance self-reliance split txt'),
|
||||
('-be_split!.txt', '-be split! txt'),
|
||||
])
|
||||
def test_split_file_name_words(name, expected):
|
||||
assert lib.split_file_name_words(name) == expected
|
||||
|
|
@ -327,11 +426,11 @@ async def test_large_text_indexer(test_session, make_files_structure):
|
|||
'large_file.txt': 'foo ' * 1_000_000,
|
||||
})
|
||||
await lib.refresh_files()
|
||||
assert test_session.query(File).count() == 1
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
assert large.is_file() and large.stat().st_size == 4_000_000
|
||||
|
||||
large_file: File = test_session.query(File).one()
|
||||
large_file: FileGroup = test_session.query(FileGroup).one()
|
||||
assert len(large_file.d_text) < large.stat().st_size
|
||||
assert len(large_file.d_text) == 90_072
|
||||
|
||||
|
|
@ -392,53 +491,81 @@ def test_matching_directories(make_files_structure, test_directory):
|
|||
assert matches == [str(test_directory / 'foo/qux')]
|
||||
|
||||
|
||||
def test_pdf_indexer(example_pdf):
|
||||
"""PDFs can be indexed by PDFIndexer."""
|
||||
file = File(path=example_pdf)
|
||||
|
||||
a_text, b_text, c_text, d_text = indexers.PDFIndexer.create_index(file)
|
||||
|
||||
# The title extracted from the PDF.
|
||||
assert a_text == 'WROLPi Test PDF'
|
||||
# The author.
|
||||
assert b_text == 'roland'
|
||||
# The parsed file name.
|
||||
assert c_text == ['example', 'pdf']
|
||||
# All pages are extracted. Text is formatted to fit on vertical screen.
|
||||
assert d_text == 'Page one\n' \
|
||||
'Page two\n' \
|
||||
'Lorem ipsum dolor sit amet,\n' \
|
||||
'consectetur adipiscing elit, sed do\n' \
|
||||
'eiusmod tempor incididunt ut labore et\n' \
|
||||
'\n' \
|
||||
'dolore magna aliqua. Ut enim ad minim\n' \
|
||||
'veniam, quis nostrud exercitation\n' \
|
||||
'ullamco laboris nisi ut \n' \
|
||||
'aliquip ex ea commodo consequat. Duis\n' \
|
||||
'aute irure dolor in reprehenderit in\n' \
|
||||
'voluptate velit esse cillum \n' \
|
||||
'dolore eu fugiat nulla pariatur.\n' \
|
||||
'Excepteur sint occaecat cupidatat non\n' \
|
||||
'proident, sunt in culpa qui officia \n' \
|
||||
'deserunt mollit anim id est laborum.'
|
||||
|
||||
|
||||
def test_pdf_indexer_max_size(example_pdf):
|
||||
"""The contents of a large PDF are not indexed."""
|
||||
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
|
||||
|
||||
file = File(path=example_pdf)
|
||||
|
||||
a_text, b_text, c_text, d_text = indexers.PDFIndexer.create_index(file)
|
||||
|
||||
# The file name is still indexed.
|
||||
assert a_text == 'WROLPi Test PDF'
|
||||
assert d_text is None
|
||||
|
||||
|
||||
def test_get_mimetype(example_epub, example_mobi, example_pdf, image_file, video_file):
|
||||
assert lib.get_mimetype(example_epub) == 'application/epub+zip'
|
||||
assert lib.get_mimetype(example_mobi) == 'application/x-mobipocket-ebook'
|
||||
assert lib.get_mimetype(example_pdf) == 'application/pdf'
|
||||
assert lib.get_mimetype(image_file) == 'image/jpeg'
|
||||
assert lib.get_mimetype(video_file) == 'video/mp4'
|
||||
|
||||
|
||||
def test_group_files_by_stem(make_files_structure, test_directory):
|
||||
make_files_structure([
|
||||
'foo.mp4',
|
||||
'foo.txt',
|
||||
'foo.info.json',
|
||||
'bar.txt',
|
||||
'baz.txt',
|
||||
])
|
||||
|
||||
files = list(test_directory.iterdir())
|
||||
assert list(lib.group_files_by_stem(files)) == [
|
||||
[test_directory / 'bar.txt'],
|
||||
[test_directory / 'baz.txt'],
|
||||
[test_directory / 'foo.info.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'],
|
||||
]
|
||||
|
||||
|
||||
def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, example_mobi, example_pdf,
|
||||
singlefile_contents_factory):
|
||||
"""Test that the most important file is returned from a list of files."""
|
||||
srt_file3 = srt_file3.rename(test_directory / f'{video_file.stem}.srt')
|
||||
# Same primary_file no matter the order.
|
||||
assert lib.get_primary_file([video_file, srt_file3]) == video_file
|
||||
assert lib.get_primary_file([srt_file3, video_file]) == video_file
|
||||
|
||||
singlefile = test_directory / 'singlefile.html'
|
||||
singlefile.write_text(singlefile_contents_factory())
|
||||
singlefile_text = test_directory / 'singlefile.txt'
|
||||
singlefile_text.write_text('the contents')
|
||||
|
||||
assert lib.get_primary_file([singlefile, singlefile_text]) == singlefile
|
||||
|
||||
assert lib.get_primary_file([example_epub, example_mobi]) == example_epub
|
||||
assert lib.get_primary_file([example_mobi, example_epub]) == example_epub
|
||||
|
||||
assert lib.get_primary_file([example_pdf, example_mobi]) == example_pdf
|
||||
assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub
|
||||
|
||||
|
||||
COMMON_JOIN = '''LEFT JOIN tag_file tf ON tf.file_group_id = fg.id
|
||||
LEFT JOIN tag t ON t.id = tf.tag_id'''
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'tags,wheres,params,join', [
|
||||
(
|
||||
['one'],
|
||||
'(t.name = %(tag_name0)s)',
|
||||
dict(tag_name0='one'),
|
||||
COMMON_JOIN,
|
||||
),
|
||||
(
|
||||
['one', 'two'],
|
||||
'(t.name = %(tag_name0)s OR t.name = %(tag_name1)s)',
|
||||
dict(tag_name0='one', tag_name1='two'),
|
||||
COMMON_JOIN,
|
||||
),
|
||||
(
|
||||
[],
|
||||
'',
|
||||
dict(),
|
||||
'',
|
||||
),
|
||||
]
|
||||
)
|
||||
def test_tag_names_to_clauses(tags, wheres, params, join):
|
||||
result = lib.tag_names_to_clauses(tags)
|
||||
assert result[0] == wheres
|
||||
assert result[1] == params
|
||||
assert result[2] == join
|
||||
|
|
|
|||
|
|
@ -2,18 +2,71 @@ from unittest import mock
|
|||
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files import pdfs
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_title(test_session, example_pdf):
|
||||
"""PDF modeler extracts the PDF's title."""
|
||||
await refresh_files()
|
||||
"""PDF modeler extracts a PDF's title and contents."""
|
||||
await files_lib.refresh_files()
|
||||
|
||||
with mock.patch('wrolpi.files.pdfs.get_pdf_title') as mock_get_pdf_title:
|
||||
mock_get_pdf_title.side_effect = Exception('should not be read again')
|
||||
await refresh_files()
|
||||
with mock.patch('wrolpi.files.pdfs.create_index') as mock_create_index:
|
||||
mock_create_index.side_effect = Exception('Should not index twice')
|
||||
await files_lib.refresh_files()
|
||||
|
||||
pdf: File = test_session.query(File).one()
|
||||
assert pdf.title == 'WROLPi Test PDF'
|
||||
pdf: FileGroup = test_session.query(FileGroup).one()
|
||||
assert pdf.indexed is True
|
||||
assert pdf.title == pdf.a_text == 'WROLPi Test PDF'
|
||||
assert pdf.b_text == 'roland'
|
||||
assert pdf.c_text == 'example pdf'
|
||||
assert pdf.d_text and pdf.d_text.startswith('Page one\n') and len(pdf.d_text) == 467
|
||||
assert pdf.data and pdf.data.get('author') == 'roland'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_modeler(test_session, example_pdf):
|
||||
"""PDFs can be indexed by PDFIndexer."""
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
|
||||
await pdfs.pdf_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
a_text, b_text, c_text, d_text = file_group.a_text, file_group.b_text, file_group.c_text, file_group.d_text
|
||||
|
||||
# The title extracted from the PDF.
|
||||
assert a_text == 'WROLPi Test PDF'
|
||||
# The author.
|
||||
assert b_text == 'roland'
|
||||
# The parsed file name.
|
||||
assert c_text == 'example pdf'
|
||||
# All pages are extracted. Text is formatted to fit on vertical screen.
|
||||
assert d_text == 'Page one\n' \
|
||||
'Page two\n' \
|
||||
'Lorem ipsum dolor sit amet,\n' \
|
||||
'consectetur adipiscing elit, sed do\n' \
|
||||
'eiusmod tempor incididunt ut labore et\n' \
|
||||
'\n' \
|
||||
'dolore magna aliqua. Ut enim ad minim\n' \
|
||||
'veniam, quis nostrud exercitation\n' \
|
||||
'ullamco laboris nisi ut \n' \
|
||||
'aliquip ex ea commodo consequat. Duis\n' \
|
||||
'aute irure dolor in reprehenderit in\n' \
|
||||
'voluptate velit esse cillum \n' \
|
||||
'dolore eu fugiat nulla pariatur.\n' \
|
||||
'Excepteur sint occaecat cupidatat non\n' \
|
||||
'proident, sunt in culpa qui officia \n' \
|
||||
'deserunt mollit anim id est laborum.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_indexer_max_size(test_session, example_pdf):
|
||||
"""The contents of a large PDF are not indexed."""
|
||||
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
|
||||
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
|
||||
await pdfs.pdf_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
assert file_group.d_text is None
|
||||
|
|
|
|||
185
wrolpi/lang.py
Normal file
185
wrolpi/lang.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
ISO_639_CODES = [
|
||||
'aa', # Afar
|
||||
'ab', # Abkhazian
|
||||
'ae', # Avestan
|
||||
'af', # Afrikaans
|
||||
'ak', # Akan
|
||||
'am', # Amharic
|
||||
'an', # Aragonese
|
||||
'ar', # Arabic
|
||||
'as', # Assamese
|
||||
'av', # Avaric
|
||||
'ay', # Aymara
|
||||
'az', # Azerbaijani
|
||||
'ba', # Bashkir
|
||||
'be', # Belarusian
|
||||
'bg', # Bulgarian
|
||||
'bi', # Bislama
|
||||
'bm', # Bambara
|
||||
'bn', # Bengali
|
||||
'bo', # Tibetan
|
||||
'br', # Breton
|
||||
'bs', # Bosnian
|
||||
'ca', # Catalan, Valencian
|
||||
'ce', # Chechen
|
||||
'ch', # Chamorro
|
||||
'co', # Corsican
|
||||
'cr', # Cree
|
||||
'cs', # Czech
|
||||
'cu', # Church Slavonic, Old Slavonic, Old Church Slavonic
|
||||
'cv', # Chuvash
|
||||
'cy', # Welsh
|
||||
'da', # Danish
|
||||
'de', # German
|
||||
'dv', # Divehi, Dhivehi, Maldivian
|
||||
'dz', # Dzongkha
|
||||
'ee', # Ewe
|
||||
'el', # Greek, Modern (1453–)
|
||||
'en', # English
|
||||
'eo', # Esperanto
|
||||
'es', # Spanish, Castilian
|
||||
'et', # Estonian
|
||||
'eu', # Basque
|
||||
'fa', # Persian
|
||||
'ff', # Fulah
|
||||
'fi', # Finnish
|
||||
'fj', # Fijian
|
||||
'fo', # Faroese
|
||||
'fr', # French
|
||||
'fy', # Western Frisian
|
||||
'ga', # Irish
|
||||
'gd', # Gaelic, Scottish Gaelic
|
||||
'gl', # Galician
|
||||
'gn', # Guarani
|
||||
'gu', # Gujarati
|
||||
'gv', # Manx
|
||||
'ha', # Hausa
|
||||
'he', # Hebrew
|
||||
'hi', # Hindi
|
||||
'ho', # Hiri Motu
|
||||
'hr', # Croatian
|
||||
'ht', # Haitian, Haitian Creole
|
||||
'hu', # Hungarian
|
||||
'hy', # Armenian
|
||||
'hz', # Herero
|
||||
'ia', # Interlingua (International Auxiliary Language Association)
|
||||
'id', # Indonesian
|
||||
'ie', # Interlingue, Occidental
|
||||
'ig', # Igbo
|
||||
'ii', # Sichuan Yi, Nuosu
|
||||
'ik', # Inupiaq
|
||||
'io', # Ido
|
||||
'is', # Icelandic
|
||||
'it', # Italian
|
||||
'iu', # Inuktitut
|
||||
'ja', # Japanese
|
||||
'jv', # Javanese
|
||||
'ka', # Georgian
|
||||
'kg', # Kongo
|
||||
'ki', # Kikuyu, Gikuyu
|
||||
'kj', # Kuanyama, Kwanyama
|
||||
'kk', # Kazakh
|
||||
'kl', # Kalaallisut, Greenlandic
|
||||
'km', # Central Khmer
|
||||
'kn', # Kannada
|
||||
'ko', # Korean
|
||||
'kr', # Kanuri
|
||||
'ks', # Kashmiri
|
||||
'ku', # Kurdish
|
||||
'kv', # Komi
|
||||
'kw', # Cornish
|
||||
'ky', # Kirghiz, Kyrgyz
|
||||
'la', # Latin
|
||||
'lb', # Luxembourgish, Letzeburgesch
|
||||
'lg', # Ganda
|
||||
'li', # Limburgan, Limburger, Limburgish
|
||||
'ln', # Lingala
|
||||
'lo', # Lao
|
||||
'lt', # Lithuanian
|
||||
'lu', # Luba-Katanga
|
||||
'lv', # Latvian
|
||||
'mg', # Malagasy
|
||||
'mh', # Marshallese
|
||||
'mi', # Maori
|
||||
'mk', # Macedonian
|
||||
'ml', # Malayalam
|
||||
'mn', # Mongolian
|
||||
'mr', # Marathi
|
||||
'ms', # Malay
|
||||
'mt', # Maltese
|
||||
'my', # Burmese
|
||||
'na', # Nauru
|
||||
'nb', # Norwegian Bokmål
|
||||
'nd', # North Ndebele
|
||||
'ne', # Nepali
|
||||
'ng', # Ndonga
|
||||
'nl', # Dutch, Flemish
|
||||
'nn', # Norwegian Nynorsk
|
||||
'no', # Norwegian
|
||||
'nr', # South Ndebele
|
||||
'nv', # Navajo, Navaho
|
||||
'ny', # Chichewa, Chewa, Nyanja
|
||||
'oc', # Occitan
|
||||
'oj', # Ojibwa
|
||||
'om', # Oromo
|
||||
'or', # Oriya
|
||||
'os', # Ossetian, Ossetic
|
||||
'pa', # Punjabi, Panjabi
|
||||
'pi', # Pali
|
||||
'pl', # Polish
|
||||
'ps', # Pashto, Pushto
|
||||
'pt', # Portuguese
|
||||
'qu', # Quechua
|
||||
'rm', # Romansh
|
||||
'rn', # Rundi
|
||||
'ro', # Romanian, Moldavian, Moldovan
|
||||
'ru', # Russian
|
||||
'rw', # Kinyarwanda
|
||||
'sa', # Sanskrit
|
||||
'sc', # Sardinian
|
||||
'sd', # Sindhi
|
||||
'se', # Northern Sami
|
||||
'sg', # Sango
|
||||
'si', # Sinhala, Sinhalese
|
||||
'sk', # Slovak
|
||||
'sl', # Slovenian
|
||||
'sm', # Samoan
|
||||
'sn', # Shona
|
||||
'so', # Somali
|
||||
'sq', # Albanian
|
||||
'sr', # Serbian
|
||||
'ss', # Swati
|
||||
'st', # Southern Sotho
|
||||
'su', # Sundanese
|
||||
'sv', # Swedish
|
||||
'sw', # Swahili
|
||||
'ta', # Tamil
|
||||
'te', # Telugu
|
||||
'tg', # Tajik
|
||||
'th', # Thai
|
||||
'ti', # Tigrinya
|
||||
'tk', # Turkmen
|
||||
'tl', # Tagalog
|
||||
'tn', # Tswana
|
||||
'to', # Tonga (Tonga Islands)
|
||||
'tr', # Turkish
|
||||
'ts', # Tsonga
|
||||
'tt', # Tatar
|
||||
'tw', # Twi
|
||||
'ty', # Tahitian
|
||||
'ug', # Uighur, Uyghur
|
||||
'uk', # Ukrainian
|
||||
'ur', # Urdu
|
||||
'uz', # Uzbek
|
||||
've', # Venda
|
||||
'vi', # Vietnamese
|
||||
'vo', # Volapük
|
||||
'wa', # Walloon
|
||||
'wo', # Wolof
|
||||
'xh', # Xhosa
|
||||
'yi', # Yiddish
|
||||
'yo', # Yoruba
|
||||
'za', # Zhuang, Chuang
|
||||
'zh', # Chinese
|
||||
'zu', # Zulu
|
||||
]
|
||||
|
|
@ -15,6 +15,7 @@ from sanic_ext import validate
|
|||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi import admin, status, flags
|
||||
from wrolpi import tags
|
||||
from wrolpi.admin import HotspotStatus
|
||||
from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \
|
||||
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics
|
||||
|
|
@ -24,7 +25,7 @@ from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError
|
|||
from wrolpi.events import get_events
|
||||
from wrolpi.files.lib import get_file_statistics
|
||||
from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse, \
|
||||
EventsRequest
|
||||
EventsRequest, NewTagRequest, DeleteTagRequest
|
||||
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
|
||||
truthy_arg
|
||||
from wrolpi.version import __version__
|
||||
|
|
@ -32,6 +33,7 @@ from wrolpi.version import __version__
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
api_app = Sanic(name='api_app')
|
||||
api_app.config.FALLBACK_ERROR_FORMAT = 'json'
|
||||
|
||||
api_bp = Blueprint('RootAPI', url_prefix='/api')
|
||||
|
||||
|
|
@ -377,6 +379,26 @@ async def feed(_: Request, query: EventsRequest):
|
|||
return json_response(dict(events=events, now=start))
|
||||
|
||||
|
||||
@api_bp.get('/tag')
|
||||
async def get_tags_request(_: Request):
|
||||
tags_ = tags.get_tags()
|
||||
return json_response(dict(tags=tags_))
|
||||
|
||||
|
||||
@api_bp.post('/tag/new')
|
||||
@validate(NewTagRequest)
|
||||
async def post_new_tag(_: Request, body: NewTagRequest):
|
||||
tags.new_tag(body.name, body.color)
|
||||
return response.empty(HTTPStatus.CREATED)
|
||||
|
||||
|
||||
@api_bp.post('/tag/delete')
|
||||
@validate(DeleteTagRequest)
|
||||
async def delete_tag_request(_: Request, body: DeleteTagRequest):
|
||||
tags.delete_tag(body.name)
|
||||
return response.empty(HTTPStatus.NO_CONTENT)
|
||||
|
||||
|
||||
class CustomJSONEncoder(json.JSONEncoder):
|
||||
|
||||
def default(self, obj):
|
||||
|
|
@ -391,7 +413,8 @@ class CustomJSONEncoder(json.JSONEncoder):
|
|||
else:
|
||||
# A datetime with no timezone is UTC.
|
||||
obj = obj.replace(tzinfo=timezone.utc)
|
||||
return obj.isoformat()
|
||||
obj = obj.isoformat()
|
||||
return obj
|
||||
elif isinstance(obj, date):
|
||||
# API always returns dates in UTC.
|
||||
obj = datetime(obj.year, obj.month, obj.day, tzinfo=timezone.utc)
|
||||
|
|
|
|||
|
|
@ -64,3 +64,14 @@ class JSONErrorResponse:
|
|||
@dataclass
|
||||
class EventsRequest:
|
||||
after: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class NewTagRequest:
|
||||
name: str
|
||||
color: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeleteTagRequest:
|
||||
name: str
|
||||
|
|
|
|||
174
wrolpi/tags.py
Normal file
174
wrolpi/tags.py
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
import contextlib
|
||||
from typing import List
|
||||
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
|
||||
from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.errors import UnknownTag, UsedTag
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
class TagFile(ModelHelper, Base):
|
||||
__tablename__ = 'tag_file'
|
||||
|
||||
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
|
||||
tag = relationship('Tag', back_populates='tag_files')
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), primary_key=True)
|
||||
file_group = relationship('FileGroup', back_populates='tag_files')
|
||||
|
||||
|
||||
class Tag(ModelHelper, Base):
|
||||
__tablename__ = 'tag'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String, unique=True, nullable=False)
|
||||
color = Column(String)
|
||||
|
||||
tag_files = relationship('TagFile', back_populates='tag', cascade='all')
|
||||
|
||||
def __repr__(self):
|
||||
name = self.name
|
||||
color = self.color
|
||||
return f'<Tag {name=} {color=}>'
|
||||
|
||||
def __json__(self):
|
||||
return dict(
|
||||
id=self.id,
|
||||
name=self.name,
|
||||
color=self.color,
|
||||
)
|
||||
|
||||
@optional_session
|
||||
def add_tag(self, file_group, session: Session = None) -> TagFile:
|
||||
"""Add a TagFile for the provided FileGroup and this Tag.
|
||||
|
||||
@warning: Commits the session to keep the config in sync."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
if not isinstance(file_group, FileGroup):
|
||||
raise ValueError('Cannot apply tag to non-FileGroup')
|
||||
|
||||
tag_file = TagFile(file_group_id=file_group.id, tag_id=self.id)
|
||||
logger.info(f'Tagging {file_group} with {self}')
|
||||
session.add(tag_file)
|
||||
session.flush([tag_file])
|
||||
get_tags_config().add_tag(tag_file)
|
||||
session.commit()
|
||||
return tag_file
|
||||
|
||||
@optional_session
|
||||
def remove_tag(self, file_group, session: Session = None):
|
||||
"""Remove the record of a Tag applied to the FileGroup.
|
||||
|
||||
@warning: Commits the session to keep config in sync."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
if not isinstance(file_group, FileGroup):
|
||||
raise ValueError('Cannot remove tag of non-FileGroup')
|
||||
|
||||
tag_file = session.query(TagFile) \
|
||||
.filter(TagFile.file_group_id == file_group.id, TagFile.tag_id == self.id) \
|
||||
.one_or_none()
|
||||
if tag_file:
|
||||
session.delete(tag_file)
|
||||
get_tags_config().remove_tag(tag_file)
|
||||
session.commit()
|
||||
else:
|
||||
logger.warning(f'Could not find tag_file for FileGroup.id={file_group.id}/Tag.id={self.id=}')
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def find_by_name(name: str, session: Session) -> 'Tag':
|
||||
tag = session.query(Tag).filter_by(name=name).one_or_none()
|
||||
return tag
|
||||
|
||||
|
||||
class TagsConfig(ConfigFile):
|
||||
file_name = 'tags.yaml'
|
||||
|
||||
default_config = dict(
|
||||
tags=list(),
|
||||
)
|
||||
|
||||
@property
|
||||
def tags(self) -> list:
|
||||
return self._config['tags']
|
||||
|
||||
@tags.setter
|
||||
def tags(self, value):
|
||||
self.update({'tags': value})
|
||||
|
||||
def add_tag(self, tag_file: TagFile):
|
||||
from wrolpi.files.models import FileGroup
|
||||
file_group: FileGroup = tag_file.file_group
|
||||
primary_path = str(file_group.primary_path.relative_to(get_media_directory()))
|
||||
tag_name = tag_file.tag.name
|
||||
tags = self.tags.copy()
|
||||
|
||||
value = [primary_path, tag_name]
|
||||
if value not in tags:
|
||||
tags.append(value)
|
||||
self.tags = tags
|
||||
|
||||
def remove_tag(self, tag_file: TagFile):
|
||||
from wrolpi.files.models import FileGroup
|
||||
file_group: FileGroup = tag_file.file_group
|
||||
primary_path = str(file_group.primary_path.relative_to(get_media_directory()))
|
||||
tag_name = tag_file.tag.name
|
||||
tags = self.tags.copy()
|
||||
|
||||
for idx, value in enumerate(self.tags):
|
||||
if value == [primary_path, tag_name]:
|
||||
tags = tags.copy()
|
||||
tags.pop(idx)
|
||||
self.tags = tags
|
||||
break
|
||||
|
||||
|
||||
TAGS_CONFIG: TagsConfig = TagsConfig(global_=True)
|
||||
TEST_TAGS_CONFIG: TAGS_CONFIG = None
|
||||
|
||||
|
||||
def get_tags_config():
|
||||
global TEST_TAGS_CONFIG
|
||||
if isinstance(TEST_TAGS_CONFIG, ConfigFile):
|
||||
return TEST_TAGS_CONFIG
|
||||
|
||||
global TAGS_CONFIG
|
||||
return TAGS_CONFIG
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def test_tags_config():
|
||||
global TEST_TAGS_CONFIG
|
||||
TEST_TAGS_CONFIG = TagsConfig()
|
||||
yield
|
||||
TEST_TAGS_CONFIG = None
|
||||
|
||||
|
||||
@optional_session
|
||||
def get_tags(session: Session) -> List[Tag]:
|
||||
tags = list(session.query(Tag))
|
||||
return tags
|
||||
|
||||
|
||||
@optional_session
|
||||
def new_tag(name: str, color: str, session: Session) -> Tag:
|
||||
tag = Tag(name=name, color=color)
|
||||
session.add(tag)
|
||||
session.flush([tag])
|
||||
session.commit()
|
||||
return tag
|
||||
|
||||
|
||||
@optional_session
|
||||
def delete_tag(name: str, session: Session = None):
|
||||
tag: Tag = Tag.find_by_name(name, session)
|
||||
if not tag:
|
||||
raise UnknownTag(f'Cannot find tag {name}')
|
||||
if tag.tag_files:
|
||||
count = len(tag.tag_files)
|
||||
raise UsedTag(f'Cannot delete {name} it is used by {count} files!')
|
||||
|
||||
session.delete(tag)
|
||||
session.commit()
|
||||
|
|
@ -415,8 +415,7 @@ def test_chunks_by_stem(test_directory, make_files_structure):
|
|||
assert_chunks(1, files, [['1.mp4', '1.txt'], ['2.mp4', '2.png', '2.txt'], ['3.mp4']])
|
||||
assert_chunks(3, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt'], ['3.mp4']])
|
||||
assert_chunks(6, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt', '3.mp4']])
|
||||
# List is not sorted because it is less than the size.
|
||||
assert_chunks(20, files, [['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']])
|
||||
assert_chunks(20, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt', '3.mp4']])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ def assert_events(expected: List[Dict], after=None):
|
|||
assert_dict_contains(event, expected)
|
||||
|
||||
|
||||
def test_events_api_feed(test_session, test_client):
|
||||
def test_events_api_feed(test_session, test_client, example_pdf):
|
||||
"""Events can be gotten from the API."""
|
||||
# refresh_files() creates a few events.
|
||||
request, response = test_client.post('/api/files/refresh')
|
||||
|
|
@ -29,9 +29,10 @@ def test_events_api_feed(test_session, test_client):
|
|||
|
||||
expected = [
|
||||
dict(event='global_refresh_completed', subject='refresh'),
|
||||
dict(event='global_after_refresh_completed', subject='refresh'),
|
||||
dict(event='global_refresh_indexing_completed', subject='refresh'),
|
||||
dict(event='global_refresh_delete_completed', subject='refresh'),
|
||||
dict(event='global_refresh_modeling_completed', subject='refresh'),
|
||||
dict(event='global_refresh_discovery_completed', subject='refresh'),
|
||||
dict(event='global_refresh_started', subject='refresh'),
|
||||
]
|
||||
for expected, event in zip_longest(expected, result):
|
||||
|
|
|
|||
96
wrolpi/test/test_tags.py
Normal file
96
wrolpi/test/test_tags.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import pytest
|
||||
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import TagFile, get_tags_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_file_group_json(test_session, make_files_structure, tag_factory, example_pdf):
|
||||
"""The tags of a file are returned in it's JSON."""
|
||||
tag_one = tag_factory()
|
||||
tag_two = tag_factory()
|
||||
await files_lib.refresh_files()
|
||||
file_group: FileGroup = test_session.query(FileGroup).one()
|
||||
|
||||
tag_one.add_tag(file_group)
|
||||
assert file_group.__json__()['tags'] == ['one']
|
||||
|
||||
file_group.add_tag(tag_two)
|
||||
assert file_group.__json__()['tags'] == ['one', 'two']
|
||||
|
||||
file_group.remove_tag(tag_one)
|
||||
assert file_group.__json__()['tags'] == ['two']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_file_group(test_session, make_files_structure, tag_factory):
|
||||
"""A FileGroup can be tagged with multiple Tags."""
|
||||
make_files_structure(['video.mp4', 'video.png'])
|
||||
await files_lib.refresh_files()
|
||||
video_group: FileGroup = test_session.query(FileGroup).one()
|
||||
|
||||
tag_one = tag_factory()
|
||||
assert len(tag_one.tag_files) == 0
|
||||
|
||||
tag_one.add_tag(video_group)
|
||||
test_session.commit()
|
||||
assert len(tag_one.tag_files) == 1
|
||||
assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ]
|
||||
|
||||
tag_two = tag_factory()
|
||||
tag_two.add_tag(video_group)
|
||||
test_session.commit()
|
||||
assert len(tag_one.tag_files) == 1
|
||||
assert len(tag_two.tag_files) == 1
|
||||
|
||||
# Video has both tags.
|
||||
assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', 'two']
|
||||
assert test_session.query(TagFile).count() == 2
|
||||
|
||||
# Deleting the tag deletes the tag_file.
|
||||
test_session.delete(tag_one)
|
||||
test_session.commit()
|
||||
assert sorted([i.tag.name for i in video_group.tag_files]) == ['two', ]
|
||||
assert test_session.query(TagFile).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_config_(test_session, test_directory, tag_factory, example_pdf, video_file, test_tags_config):
|
||||
"""Test that the config is updated when a FileGroup is tagged."""
|
||||
await files_lib.refresh_files()
|
||||
pdf: FileGroup = FileGroup.find_by_path(example_pdf, test_session)
|
||||
video: FileGroup = FileGroup.find_by_path(video_file, test_session)
|
||||
tag1 = tag_factory()
|
||||
tag2 = tag_factory()
|
||||
|
||||
video.add_tag(tag1)
|
||||
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
|
||||
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
|
||||
|
||||
# Tag PDF twice.
|
||||
pdf.add_tag(tag1)
|
||||
pdf.add_tag(tag2)
|
||||
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
|
||||
assert str(example_pdf.relative_to(test_directory)) in test_tags_config.read_text()
|
||||
|
||||
# One TagFile still exists.
|
||||
pdf.remove_tag(tag1)
|
||||
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
|
||||
assert str(example_pdf.relative_to(test_directory)) in test_tags_config.read_text()
|
||||
|
||||
# PDF is no longer tagged.
|
||||
pdf.remove_tag(tag2)
|
||||
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
|
||||
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
|
||||
|
||||
video.remove_tag(tag1)
|
||||
assert str(video_file.relative_to(test_directory)) not in test_tags_config.read_text()
|
||||
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
|
||||
|
||||
# No more tags.
|
||||
tags_config = get_tags_config()
|
||||
assert isinstance(tags_config.tags, list) and len(tags_config.tags) == 0
|
||||
|
||||
# Removing non-existent tag does not error.
|
||||
video.remove_tag(tag1)
|
||||
|
|
@ -47,7 +47,7 @@ def truthy_arg(value: str) -> bool:
|
|||
DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
|
||||
MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
|
||||
if not MEDIA_DIRECTORY.is_dir() and not PYTEST:
|
||||
print(f'Media directory does not exist! {MEDIA_DIRECTORY}')
|
||||
print(f'Media directory does not exist! {MEDIA_DIRECTORY}', file=sys.stderr)
|
||||
|
||||
CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config'
|
||||
MODULES_DIR: Path = PROJECT_DIR / 'modules'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue