rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #92 from lrnselfreliance/feature/files-search

Browse source 
Feature/files search

Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
lrnselfreliance 2022-02-02 14:22:28 -07:00 • committed by Roland
commit ae3c351111
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
23 changed files with 649 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

53
alembic/versions/36cae041ad31_files.py Normal file
View file

@ -0,0 +1,53 @@
"""Files
Revision ID: 36cae041ad31
Revises: b97c1313f986
Create Date: 2022-02-01 11:05:57.605541
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '36cae041ad31'
down_revision = 'b97c1313f986'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('''CREATE TABLE file (
id BIGSERIAL PRIMARY KEY,
path TEXT NOT NULL,
title TEXT,
mimetype TEXT,
size INTEGER,
modification_datetime TIMESTAMPTZ,
textsearch tsvector GENERATED ALWAYS AS (
setweight(to_tsvector('english'::regconfig, title), 'A'::"char")
) stored,
idempotency TEXT
)''')
session.execute('CREATE INDEX file_path_idx ON file(path)')
session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)')
session.execute('CREATE INDEX file_textsearch_idx ON file(textsearch)')
if not DOCKERIZED:
session.execute('ALTER TABLE public.file OWNER TO wrolpi')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP INDEX IF EXISTS file_textsearch_idx')
session.execute('DROP INDEX IF EXISTS file_mimetype_idx')
session.execute('DROP INDEX IF EXISTS file_path_idx')
session.execute('DROP TABLE IF EXISTS file')

46
app/src/api.js
View file

@ -1,4 +1,4 @@
import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, VIDEOS_API} from "./components/Common";
import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, secondsToDateTime, VIDEOS_API} from "./components/Common";
import {toast} from 'react-semantic-toasts';
function timeoutPromise(ms, promise) {
@ -303,8 +303,8 @@ export async function deleteArchive(archive_id) {
export async function searchArchives(offset, limit, domain = null, searchStr = null) {
// Build a search query to retrieve a list of videos from the API
offset = offset || 0;
limit = limit || DEFAULT_LIMIT;
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
let body = {offset, limit};
if (domain) {
body['domain'] = domain;
@ -352,19 +352,16 @@ export async function postDownload(urls, downloader) {
return response;
}
export async function killDownload(download_id) {
let response = await apiPost(`${API_URI}/download/${download_id}/kill`);
return response;
}
export async function killDownloads() {
let response = await apiPost(`${API_URI}/download/kill`);
return response;
}
export async function startDownloads() {
let response = await apiPost(`${API_URI}/download/enable`);
return response;
@ -375,10 +372,45 @@ export async function getDownloaders() {
return await response.json();
}
const replaceFileDatetimes = (files) => {
for (let i = 0; i < files.length; i++) {
let file = files[i]['modified'];
if (file['modified']) {
files[i]['modified'] = secondsToDateTime(file['modified']);
}
}
return files;
}
export async function filesSearch(offset, limit, searchStr) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
const response = await apiPost(`${API_URI}/files/search`, body);
if (response.status === 200) {
let data = await response.json();
let [files, total] = [data['files'], data['totals']['files']];
files = replaceFileDatetimes(files);
return [files, total];
} else {
toast({
type: 'error',
title: 'Unable to search files',
description: 'Cannot search files. See server logs.',
time: 5000,
});
}
}
export async function refreshFiles() {
return await apiPost(`${API_URI}/files/refresh`);
}
export async function getFiles(directories) {
let body = {directories: directories || []};
let response = await apiPost(`${API_URI}/files`, body);
return await response.json();
let {files} = await response.json();
files = replaceFileDatetimes(files);
return files;
}
export async function deleteFile(file) {

6
app/src/components/Admin.js
View file

@ -443,9 +443,9 @@ class Downloads extends React.Component {
export default function Admin(props) {
const links = [
{text: 'Downloads', to: '/admin', exact: true},
{text: 'Settings', to: '/admin/settings'},
{text: 'WROL Mode', to: '/admin/wrol'},
{text: 'Downloads', to: '/admin', exact: true, key: 'admin'},
{text: 'Settings', to: '/admin/settings', key: 'settings'},
{text: 'WROL Mode', to: '/admin/wrol', key: 'wrol'},
];
return (

20
app/src/components/Archive.js
View file

@ -1,10 +1,11 @@
import React, {useState} from "react";
import {Card, Confirm, Container, Icon, Image, Placeholder} from "semantic-ui-react";
import Paginator, {
import {
CardGroupCentered,
ClearButton,
ExternalLink,
PageContainer,
Paginator,
SearchInput,
TabLinks,
textEllipsis,
@ -202,9 +203,8 @@ export function ArchivesList({archives, searchStr, syncArchive, localDeleteArchi
}
export function Archives() {
const {archivesData, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} =
useArchives();
const {archives} = archivesData;
const {archives, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} =
useArchives(20);
const syncArchive = async (url) => {
await postDownload(url, 'archive');
@ -237,11 +237,7 @@ export function Archives() {
setDomain={setDomain}
/>
<div style={{marginTop: '3em', textAlign: 'center'}}>
<Paginator
activePage={activePage}
changePage={setPage}
totalPages={totalPages}
/>
<Paginator activePage={activePage} totalPages={totalPages} onPageChange={setPage}/>
</div>
</>
)
@ -323,9 +319,9 @@ function Domains() {
export function ArchiveRoute(props) {
const links = [
{text: 'Archives', to: '/archive', exact: true},
{text: 'Domains', to: '/archive/domains'},
{text: 'Manage', to: '/archive/manage'},
{text: 'Archives', to: '/archive', exact: true, key: 'archive'},
{text: 'Domains', to: '/archive/domains', key: 'domains'},
{text: 'Manage', to: '/archive/manage', key: 'manage'},
];
return <PageContainer>
<TabLinks links={links}/>

22
app/src/components/Common.js
View file

@ -12,7 +12,7 @@ export const ARCHIVES_API = `${API_URI}/archive`;
export const OTP_API = `${API_URI}/otp`;
export const DEFAULT_LIMIT = 20;
export default class Paginator extends React.Component {
export default class OldPaginator extends React.Component {
state = {
boundaryRange: 1,
siblingRange: 2,
@ -54,6 +54,25 @@ export default class Paginator extends React.Component {
}
}
export function Paginator({activePage, onPageChange, totalPages, showFirstAndLast}) {
const handlePageChange = (e, {activePage}) => {
onPageChange(activePage);
}
return (
<Pagination
activePage={activePage}
boundaryRange={1}
onPageChange={handlePageChange}
size='mini'
siblingRange={2}
totalPages={totalPages}
firstItem={showFirstAndLast ? undefined : null}
lastItem={showFirstAndLast ? undefined : null}
/>
)
}
export function secondsToDuration(video) {
let duration = video.duration;
let hours = Math.floor(duration / 3600);
@ -582,6 +601,7 @@ export function TabLinks({links}) {
className={'item'}
exact={link.exact || false}
style={{padding: '1em'}}
key={link.key}
>
{link.text}
</NavLink>

191
app/src/components/Files.js
View file

@ -2,8 +2,10 @@ import React from "react";
import FileBrowser from 'react-keyed-file-browser';
import {Button, Confirm, Header, Icon, Modal} from "semantic-ui-react";
import 'react-keyed-file-browser/dist/react-keyed-file-browser.css';
import {deleteFile, getFiles} from "../api";
import {humanFileSize, PageContainer, secondsToDateTime} from "./Common";
import {deleteFile, refreshFiles} from "../api";
import {Paginator, humanFileSize, PageContainer, SearchInput, TabLinks} from "./Common";
import {useBrowseFiles, useSearchFiles} from "../hooks/customHooks";
import {Route} from "react-router-dom";
const icons = {
File: <Icon name='file'/>,
@ -13,107 +15,142 @@ const icons = {
PDF: <Icon name='file pdf'/>,
};
class Files extends React.Component {
constructor(props) {
super(props);
this.state = {
deleteOpen: false,
files: [],
openFolders: [],
previewOpen: false,
selectedFile: null,
};
}
export function Files() {
const {searchFiles, setPage, totalPages, activePage, searchStr, setSearchStr} = useSearchFiles(50);
const {browseFiles, setOpenFolders, fetchFiles} = useBrowseFiles();
async componentDidMount() {
await this.fetchFiles();
}
const [deleteOpen, setDeleteOpen] = React.useState(false);
const [previewOpen, setPreviewOpen] = React.useState(false);
const [selectedFile, setSelectedFile] = React.useState(null);
fetchFiles = async () => {
let {files} = await getFiles(this.state.openFolders);
for (let i = 0; i < files.length; i++) {
files[i]['modified'] = secondsToDateTime(files[i]['modified']);
}
this.setState({files: files});
}
handleFolderChange = async (file, browserProps) => {
const handleFolderChange = async (file, browserProps) => {
let openFolders = Object.keys(browserProps.openFolders);
this.setState({openFolders}, this.fetchFiles);
setOpenFolders(openFolders);
}
onDownloadFile = async () => {
const onDownloadFile = async () => {
// Open file in new tab/window.
window.open(`/media/${this.state.selectedFile.key}`);
window.open(`/media/${selectedFile.key}`);
}
onSelectFile = async (file) => {
this.setState({selectedFile: file}, this.openPreview);
const onSelectFile = async (file) => {
setSelectedFile(file);
openPreview();
}
onDeleteFile = async () => {
await deleteFile(this.state.selectedFile.key);
await this.fetchFiles();
this.closePreview();
const onDeleteFile = async () => {
await deleteFile(selectedFile.key);
await fetchFiles();
closePreview();
}
openPreview = () => {
this.setState({previewOpen: true});
const openPreview = () => {
setPreviewOpen(true);
}
closePreview = () => {
this.setState({previewOpen: false, selectedFile: null, deleteOpen: false});
const closePreview = () => {
setPreviewOpen(false);
setSelectedFile(null);
setDeleteOpen(false);
}
openDelete = () => {
this.setState({deleteOpen: true});
const openDelete = () => {
setDeleteOpen(true);
}
closeDelete = () => {
this.setState({deleteOpen: false});
const closeDelete = () => {
setDeleteOpen(false);
}
render() {
return <>
{this.state.selectedFile &&
<Modal closeIcon
open={this.state.previewOpen}
onClose={this.closePreview}
>
<Modal.Header><Icon name='file'/>File Preview</Modal.Header>
<Modal.Content>
<Header>{this.state.selectedFile.name}</Header>
<p>{humanFileSize(this.state.selectedFile.size)}</p>
</Modal.Content>
<Modal.Actions>
<Button color='red' onClick={this.openDelete} floated='left'>Delete</Button>
<Confirm
open={this.state.deleteOpen}
content={`Are you sure you want to delete ${this.state.selectedFile.key}`}
onCancel={this.closeDelete}
onConfirm={this.onDeleteFile}
/>
<Button color='green' onClick={this.onDownloadFile}>Download</Button>
</Modal.Actions>
</Modal>}
<FileBrowser
showActionBar={false}
canFilter={false}
files={this.state.files}
icons={icons}
onFolderOpen={this.handleFolderChange}
onFolderClose={this.handleFolderChange}
onSelectFile={this.onSelectFile}
detailRenderer={() => <></>} // Hide the preview that the 3rd party provided.
/>
</>
let selectedModal = <></>;
if (selectedFile) {
selectedModal = (
<Modal closeIcon
open={previewOpen}
onClose={closePreview}
>
<Modal.Header><Icon name='file'/>File Preview</Modal.Header>
<Modal.Content>
<Header>{selectedFile.name}</Header>
<p>{humanFileSize(selectedFile.size)}</p>
</Modal.Content>
<Modal.Actions>
<Button color='red' onClick={openDelete} floated='left'>Delete</Button>
<Confirm
open={deleteOpen}
content={`Are you sure you want to delete ${selectedFile.key}`}
onCancel={closeDelete}
onConfirm={onDeleteFile}
/>
<Button color='green' onClick={onDownloadFile}>Download</Button>
</Modal.Actions>
</Modal>);
}
let browser;
let pagination = null;
if (searchStr) {
browser = (<FileBrowser
showActionBar={false}
canFilter={false}
files={searchFiles}
icons={icons}
onSelectFile={onSelectFile}
detailRenderer={() => <></>} // Hide the preview that the 3rd party provided.
/>);
pagination = (
<div style={{marginTop: '3em', textAlign: 'center'}}>
<Paginator
activePage={activePage}
changePage={setPage}
totalPages={totalPages}
/>
</div>);
} else {
browser = (<FileBrowser
showActionBar={false}
canFilter={false}
files={browseFiles}
icons={icons}
onFolderOpen={handleFolderChange}
onFolderClose={handleFolderChange}
onSelectFile={onSelectFile}
detailRenderer={() => <></>} // Hide the preview that the 3rd party provided.
/>);
}
return <>
<SearchInput initValue={searchStr} onSubmit={setSearchStr}/>
{selectedModal}
{browser}
{pagination}
</>
}
function ManageFiles() {
return (<>
<Button secondary
id='refresh_files'
onClick={refreshFiles}>
Refresh Files
</Button>
<label htmlFor='refresh_files'>
Find and index any new files.
</label>
</>)
}
export function FilesRoute(props) {
const links = [
{text: 'Files', to: '/files', exact: true, key: 'files'},
{text: 'Manage', to: '/files/manage', exact: true, key: 'manage'},
];
return (
<PageContainer>
<Files/>
<TabLinks links={links}/>
<Route path='/files' exact component={Files}/>
<Route path='/files/manage' exact component={ManageFiles}/>
</PageContainer>
);
}

4
app/src/components/Inventory.js
View file

@ -1090,8 +1090,8 @@ class InventoryAddList extends React.Component {
export function InventoryRoute(props) {
const links = [
{text: 'List', to: '/inventory', exact: true},
{text: 'Summary', to: '/inventory/summary'},
{text: 'List', to: '/inventory', exact: true, key: 'inventory'},
{text: 'Summary', to: '/inventory/summary', key: 'summary'},
];
return (
<PageContainer>

24
app/src/components/Nav.js
View file

@ -5,17 +5,17 @@ import {Dropdown, Menu, Responsive} from "semantic-ui-react";
const responsiveWidth = 500;
const links = [
{text: 'Videos', to: '/videos'},
{text: 'Inventory', to: '/inventory'},
{text: 'Archive', to: '/archive'},
{text: 'Files', to: '/files'},
{text: 'Videos', to: '/videos', key: 'videos'},
{text: 'Inventory', to: '/inventory', key: 'inventory'},
{text: 'Archive', to: '/archive', key: 'archive'},
{text: 'Files', to: '/files', key: 'files'},
{
text: 'Apps', links: [
key: 'apps', text: 'Apps', links: [
{to: '/apps/otp', text: 'One Time Pad', exact: true},
]
},
];
const admin = {to: '/admin', text: 'Admin', exact: true};
const admin = {to: '/admin', text: 'Admin', exact: true, key: 'admin'};
const rightLinks = [admin,];
const collapsedLinks = links.concat([admin,]);
@ -67,22 +67,20 @@ export function NavBar() {
{/*Show the links in a menu when on desktop*/}
{links.map((link) => {
return (
<Responsive minWidth={responsiveWidth} as={MenuLink} link={link} key={link.to}/>
<Responsive minWidth={responsiveWidth} as={MenuLink} link={link} key={link.key}/>
)
})}
<Responsive minWidth={responsiveWidth} as={Menu.Menu} position="right">
{rightLinks.map((link) => {
return (<MenuLink link={link} key={link.to}/>)
})}
{rightLinks.map((link) => <MenuLink link={link} key={link.key}/>)}
</Responsive>
{/*Show the menu items in a dropdown when on mobile*/}
<Responsive as={Menu.Menu} maxWidth={responsiveWidth - 1} position='right'>
<Dropdown item icon="bars">
<Dropdown.Menu>
{collapsedLinks.map((link) => {
return (<MenuLink link={link} key={link.to}/>)
})}
{collapsedLinks.map((link) =>
<MenuLink link={link} key={link.key}/>
)}
</Dropdown.Menu>
</Dropdown>
</Responsive>

12
app/src/components/Videos.js
View file

@ -1,6 +1,6 @@
import React from 'react';
import {Link, Route} from "react-router-dom";
import Paginator, {
import OldPaginator, {
changePageHistory,
DEFAULT_LIMIT,
defaultSearchOrder,
@ -401,7 +401,7 @@ class Videos extends React.Component {
if (totalPages) {
pagination = (
<div style={{marginTop: '3em', textAlign: 'center'}}>
<Paginator
<OldPaginator
activePage={activePage}
changePage={this.changePage}
totalPages={totalPages}
@ -496,10 +496,10 @@ function ManageTab(props) {
export function VideosRoute(props) {
const links = [
{text: 'Videos', to: '/videos', exact: true},
{text: 'Favorites', to: '/videos/favorites'},
{text: 'Channels', to: '/videos/channel',},
{text: 'Manage', to: '/videos/manage'},
{text: 'Videos', to: '/videos', exact: true, key: 'videos'},
{text: 'Favorites', to: '/videos/favorites', key: 'favorites'},
{text: 'Channels', to: '/videos/channel', key: 'channel'},
{text: 'Manage', to: '/videos/manage', key: 'manage'},
];
return (

90
app/src/hooks/customHooks.js
View file

@ -1,5 +1,5 @@
import {useEffect, useState} from "react";
import {fetchDomains, getVersion, searchArchives, searchVideos} from "../api";
import {fetchDomains, filesSearch, getFiles, getVersion, searchArchives, searchVideos} from "../api";
import {useHistory} from "react-router-dom";
export const useSearchParam = (key, defaultValue = null) => {
@ -68,15 +68,26 @@ export const useDomains = () => {
return [domains, total];
}
export const useArchives = (defaultLimit = 20) => {
const [archivesData, setArchives] = useState({archives: null, total: 0});
const [totalPages, setTotalPages] = useState(0);
const [activePage, setActivePage] = useState(1);
let [offset, setOffset] = useSearchParam('o');
export const useArchives = ({defaultLimit = 20}) => {
let [limit, setLimit] = useSearchParam('l', defaultLimit);
let [domain, setDomain] = useSearchParam('domain');
let [offset, setOffset] = useSearchParam('o');
let [searchStr, setSearchStr] = useSearchParam('q');
let [domain, setDomain] = useSearchParam('domain');
const [archives, setArchives] = useState(null);
const [totalPages, setTotalPages] = useState(0);
const [activePage, setActivePage] = useState((offset / limit) + 1);
const search = async () => {
setArchives(null);
const [archives, total] = await searchArchives(offset, limit, domain, searchStr);
setTotalPages(Math.floor(total / limit) + 1);
setArchives(archives);
}
useEffect(() => {
search();
}, [offset, limit, domain, searchStr]);
const setPage = (i) => {
i = parseInt(i);
@ -85,20 +96,8 @@ export const useArchives = (defaultLimit = 20) => {
setActivePage(i);
}
const search = async (term) => {
setArchives({archives: null, total: 0});
setTotalPages(0);
const [archives, total] = await searchArchives(offset, limit, domain, term);
setTotalPages(Math.floor(total / limit) + 1);
setArchives({archives, total});
}
useEffect(() => {
search(searchStr);
}, [offset, limit, domain, searchStr]);
return {
archivesData,
archives,
totalPages,
setTotalPages,
offset,
@ -130,3 +129,52 @@ export const useVersion = () => {
return version;
}
export const useSearchFiles = ({defaultLimit = 50}) => {
let [limit, setLimit] = useSearchParam('l', defaultLimit);
let [offset, setOffset] = useSearchParam('o');
let [searchStr, setSearchStr] = useSearchParam('q');
const [searchFiles, setSearchFiles] = useState([]);
const [totalPages, setTotalPages] = useState(0);
const [activePage, setActivePage] = useState(1);
const localSearchFiles = async () => {
setSearchFiles([]);
setTotalPages(0);
if (searchStr) {
let [files, total] = await filesSearch(offset, limit, searchStr);
setSearchFiles(files);
setTotalPages(Math.floor(total / limit) + 1);
}
}
useEffect(() => {
localSearchFiles();
}, [searchStr, limit, offset, activePage]);
const setPage = (i) => {
i = parseInt(i);
let l = parseInt(limit);
setOffset((l * i) - l);
setActivePage(i);
}
return {searchFiles, totalPages, limit, setLimit, setOffset, searchStr, setSearchStr, activePage, setPage};
}
export const useBrowseFiles = () => {
const [browseFiles, setBrowseFiles] = useState([]);
const [openFolders, setOpenFolders] = useState([]);
const fetchFiles = async () => {
const files = await getFiles(openFolders);
setBrowseFiles(files);
}
useEffect(() => {
fetchFiles();
}, [openFolders])
return {browseFiles, openFolders, setOpenFolders, fetchFiles};
}

20
modules/files/api.py
View file

@ -6,7 +6,7 @@ from sanic import response
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import get_media_directory
from wrolpi.common import get_media_directory, api_param_limiter
from wrolpi.errors import InvalidFile
from wrolpi.root_api import get_blueprint, json_response
from . import lib, schema
@ -61,3 +61,21 @@ async def delete_file(_: Request, body: schema.DeleteRequest):
raise InvalidFile('file cannot be empty')
lib.delete_file(body.file)
return response.empty()
files_limit_limiter = api_param_limiter(100)
files_offset_limiter = api_param_limiter(100, 0)
@bp.post('/refresh')
@openapi.description('Find and index all files')
async def refresh(_: Request):
lib.refresh_files()
return response.empty()
@bp.post('/search')
@validate(schema.FilesSearchRequest)
async def search_files(_: Request, body: schema.FilesSearchRequest):
files, total = lib.search(body.search_str, body.limit, body.offset)
return json_response(dict(files=files, totals=dict(files=total)))

116
modules/files/lib.py
View file

@ -1,8 +1,24 @@
import asyncio
import re
from functools import wraps
from pathlib import Path
from typing import List
from uuid import uuid4
from wrolpi.common import get_media_directory, wrol_mode_check
import magic
import psycopg2
from sqlalchemy.orm import Session
from modules.files.models import File
from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks, logger
from wrolpi.dates import from_timestamp
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidFile
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
__all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search']
def filter_parent_directories(directories: List[Path]) -> List[Path]:
@ -56,3 +72,101 @@ def delete_file(file: str):
if file.is_dir() or not file.is_file():
raise InvalidFile(f'Invalid file {file}')
file.unlink()
FILE_NAME_REGEX = re.compile(r'[_ ]')
def split_file_name(path: Path) -> List[str]:
"""
Split a file name into words.
>>> split_file_name(Path('foo.mp4'))
['foo']
>>> split_file_name(Path('foo bar.mp4'))
['foo', 'bar']
"""
words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)]
return words
def upsert_file(path: Path, session: Session) -> File:
"""Update/insert a File in the DB. Gather metadata about it."""
file = session.query(File).filter_by(path=path).one_or_none()
if not file:
file = File(path=path)
session.add(file)
if not file.mimetype:
file.mimetype = magic.from_file(str(path), mime=True)
if not file.title:
file.title = split_file_name(path)
if not file.size or not file.modification_datetime:
stat = path.stat()
file.size = stat.st_size
file.modification_datetime = file.modification_datetime or from_timestamp(stat.st_mtime)
return file
def _refresh_files():
"""Find and index all files"""
logger.info('Refreshing Files')
# Mark all files as unverified. Any records that are unverified after refresh will be deleted.
with get_db_curs(commit=True) as curs:
curs.execute('UPDATE file SET idempotency=null')
paths = filter(lambda i: i.is_file(), walk(get_media_directory()))
idempotency = str(uuid4())
for chunk in chunks(paths, 20):
with get_db_session(commit=True) as session:
for path in chunk:
file = upsert_file(path, session)
file.idempotency = idempotency
# Remove any records where the file no longer exists.
with get_db_curs(commit=True) as curs:
curs.execute('DELETE FROM file WHERE idempotency IS NULL')
logger.info('Done refreshing Files')
@wraps(_refresh_files)
def refresh_files():
"""Schedule a refresh task if not testing. If testing, do a synchronous refresh."""
if PYTEST:
return _refresh_files()
async def _():
return _refresh_files()
asyncio.create_task(_())
def search(search_str: str, limit: int, offset: int):
"""
Search the "title" of each file. The title is the words in the file's stem.
"""
with get_db_curs() as curs:
stmt = '''
SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total
FROM file
WHERE textsearch @@ websearch_to_tsquery(%(search_str)s)
OFFSET %(offset)s LIMIT %(limit)s
'''
params = dict(search_str=search_str, offset=offset, limit=limit)
curs.execute(stmt, params)
try:
results = list(map(dict, curs.fetchall()))
except psycopg2.ProgrammingError:
# No files.
return [], 0
total = results[0]['total'] if results else 0
ranked_ids = [i['id'] for i in results]
with get_db_session() as session:
results = get_ranked_models(ranked_ids, File, session)
results = [i.__json__() for i in results]
return results, total

36
modules/files/models.py Normal file
View file

@ -0,0 +1,36 @@
from sqlalchemy import Integer, Column, String, Computed
from sqlalchemy.orm import deferred
from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector
from wrolpi.dates import TZDateTime
from wrolpi.media_path import MediaPathType
class File(ModelHelper, Base):
__tablename__ = 'file'
id = Column(Integer, primary_key=True)
path = Column(MediaPathType)
idempotency = Column(String)
mimetype = Column(String)
modification_datetime = Column(TZDateTime)
size = Column(Integer)
title = deferred(Column(String)) # used by textsearch column
textsearch = deferred(
Column(tsvector, Computed('''to_tsvector('english'::regconfig, title)''')))
def __repr__(self):
return f'<File id={self.id} path={self.path.relative_to(get_media_directory())} mime={self.mimetype}>'
def __json__(self):
path = self.path.path.relative_to(get_media_directory())
d = dict(
id=self.id,
mimetype=self.mimetype,
modified=self.modification_datetime, # React browser expects this.
path=path,
size=self.size,
key=path,
)
return d

9
modules/files/schema.py
View file

@ -1,5 +1,5 @@
from dataclasses import dataclass, field
from typing import List
from typing import List, Optional
@dataclass
@ -10,3 +10,10 @@ class FilesRequest:
@dataclass
class DeleteRequest:
file: str
@dataclass
class FilesSearchRequest:
search_str: str
limit: Optional[int] = 20
offset: Optional[int] = 0

43
modules/files/test/test_api.py
View file

@ -6,6 +6,8 @@ from unittest import mock
import pytest
from wrolpi.errors import API_ERRORS, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains
from wrolpi.vars import PROJECT_DIR
def test_list_files_api(test_client, make_files_structure, test_directory):
@ -114,3 +116,44 @@ def test_delete_wrol_mode(test_client):
request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'}))
assert response.status_code == HTTPStatus.FORBIDDEN
assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code']
def do_search(test_client, search_str, total, expected):
content = json.dumps({'search_str': search_str})
request, response = test_client.post('/api/files/search', content=content)
assert response.json['totals']['files'] == total
for file, expected in zip_longest(response.json['files'], expected):
assert_dict_contains(file, expected)
# FileBrowser in React requires a key. We use the path.
assert file['path'] == file['key']
def test_files_search(test_session, test_client, make_files_structure):
# You can search an empty directory.
do_search(test_client, 'nothing', 0, [])
# Create files in the temporary directory. Add some contents so the mimetype can be tested.
files = [
'foo_is_the_name.txt',
'archives/bar.txt',
'baz.mp4',
'baz two.mp4'
]
foo, bar, baz, baz2 = make_files_structure(files)
foo.write_text('foo contents')
bar.write_text('the bar contents')
baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
# Refresh so files can be searched.
request, response = test_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)])
do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)])
do_search(test_client, 'baz', 2, [
dict(path='baz two.mp4', mimetype='video/mp4', size=1055736),
dict(path='baz.mp4', mimetype='video/mp4', size=1055736),
])
do_search(test_client, 'two', 1, [dict(path='baz two.mp4', mimetype='video/mp4', size=1055736)])
do_search(test_client, 'nothing', 0, [])

47
modules/files/test/test_lib.py
View file

@ -1,9 +1,12 @@
from pathlib import Path
from typing import List, Iterable
import pytest
from modules.files import lib
from modules.files.models import File
from wrolpi.errors import InvalidFile
from wrolpi.media_path import MediaPathType
def test_list_files(make_files_structure, test_directory):
@ -122,3 +125,47 @@ def test_delete_file(make_files_structure, test_directory):
lib.delete_file('does not exist')
assert (test_directory / 'baz').is_dir()
@pytest.mark.parametrize(
'path,expected',
[
('foo.mp4', ['foo']),
('foo bar.txt', ['foo', 'bar']),
('foo_bar.mp4', ['foo', 'bar']),
('foo-bar', ['foo-bar']),
('123 foo bar', ['123', 'foo', 'bar']),
('123foo bar', ['123foo', 'bar']),
]
)
def test_split_file_name(path, expected):
assert lib.split_file_name(Path(path)) == expected
def test_refresh_files(test_session, make_files_structure, test_directory):
files = make_files_structure([
'foo.txt',
'bar.txt',
'baz.txt',
])
foo, bar, baz = files
def get_relative_strs(files_: Iterable[MediaPathType]) -> List[str]:
return sorted([str(i.path.path.relative_to(test_directory)) for i in files_])
lib.refresh_files()
results = test_session.query(File)
assert get_relative_strs(results) == ['bar.txt', 'baz.txt', 'foo.txt']
baz.unlink()
lib.refresh_files()
results = test_session.query(File)
assert get_relative_strs(results) == ['bar.txt', 'foo.txt']
foo.unlink()
bar.unlink()
lib.refresh_files()
results = test_session.query(File)
assert get_relative_strs(results) == []

20
modules/videos/captions.py
View file

@ -1,5 +1,4 @@
#! /usr/bin/env python3
import pathlib
from pathlib import Path
from typing import Generator, Union
@ -7,14 +6,12 @@ import srt
import webvtt
from wrolpi.common import logger
from wrolpi.media_path import MediaPath
from .models import Video
__all__ = ['get_captions']
def get_caption_text(caption_path: Union[str, Path]) -> Generator:
"""
Return all text from each caption of a caption file.
"""
"""Return all text from each caption of a caption file."""
if str(caption_path).endswith('vtt'):
for caption in webvtt.read(caption_path):
text = str(caption.text).strip()
@ -38,15 +35,8 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator:
yield line
def get_video_captions(video: Video):
"""
Parse video captions from the video's captions file.
"""
if isinstance(video.caption_path, MediaPath):
caption_path = video.caption_path.path
else:
caption_path = pathlib.Path(video.caption_path)
def get_captions(caption_path: Path):
"""Parse video captions from the video's captions file."""
try:
lines = get_unique_caption_lines(str(caption_path))
block = '\n'.join(lines)

4
modules/videos/lib.py
View file

@ -14,7 +14,7 @@ from wrolpi.dates import from_timestamp, Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.media_path import MediaPath
from wrolpi.vars import PYTEST
from .captions import get_video_captions
from .captions import get_captions
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \
get_video_duration, is_valid_poster, convert_image, generate_video_poster
from .models import Channel, Video
@ -188,7 +188,7 @@ def validate_video(video: Video, channel_generate_poster: bool):
# Video duration was not in the info json, use ffprobe.
video.duration = get_video_duration(video_path)
if not video.caption and video.caption_path:
video.caption = get_video_captions(video)
video.caption = get_captions(video.caption_path.path)
if not video.size or not video.modification_datetime:
stat = video_path.stat()

15
modules/videos/test/test_captions.py
View file

@ -1,3 +1,4 @@
import shutil
from pathlib import Path
from modules.videos import captions
@ -46,11 +47,11 @@ class TestCaption(TestAPI):
video1 = Video(title='scream', caption_path=str(self.vtt_path1))
session.add(video1)
session.commit()
video1.caption = captions.get_video_captions(video1)
video1.caption = captions.get_captions(video1.caption_path.path)
video2 = Video(title='bar', caption_path=str(self.vtt_path2))
session.add(video2)
session.commit()
video2.caption = captions.get_video_captions(video2)
video2.caption = captions.get_captions(video2.caption_path.path)
session.commit()
session.flush()
@ -91,3 +92,13 @@ class TestCaption(TestAPI):
# Matches video2.title
select_textsearch('bar')
self.assertEqual(curs.fetchall(), [[2, ]])
def test_bad_caption(test_directory):
"""A caption file that cannot be parsed is ignored."""
bad_caption_file = PROJECT_DIR / 'test/bad_caption.en.vtt'
test_file = test_directory / 'bad_caption.en.vtt'
shutil.copy(bad_caption_file, test_file)
result = captions.get_captions(test_file)
assert result is None

3
requirements.txt
View file

@ -7,6 +7,7 @@ Pint==0.18
psycopg2-binary==2.9.3
pytest-asyncio==0.17.2
pytest==6.2.5
python-magic==0.4.24
pytz==2021.3
PyYAML==6.0
requests==2.27.1
@ -15,7 +16,7 @@ sanic-testing==0.8.2
sanic==21.12.1
selenium==4.1.0
SQLAlchemy==1.3.22
srt==3.5.0
srt==3.5.1
websockets==10.1
webvtt-py==0.4.6
yt-dlp==2022.1.21

8
test/bad_caption.en.vtt Normal file
View file

@ -0,0 +1,8 @@
WEBVTT
Kind: captions
Language: en
00:00:00.070 --> 00:00:06.380
this caption is bad
00:05:08.650 --> 00:05:11.26

8
wrolpi/common.py
View file

@ -638,3 +638,11 @@ def zig_zag(low: ZIG_TYPE, high: ZIG_TYPE) -> Generator[ZIG_TYPE, None, None]:
if num >= high:
divisor *= 2
num = low + (diff / divisor)
def walk(path: Path) -> Generator[Path, None, None]:
"""Walk a directory structure yielding all files and directories."""
for path in path.iterdir():
yield path
if path.is_dir():
yield from walk(path)

30
wrolpi/test/common.py
View file

@ -76,19 +76,7 @@ class PytestCase:
@classmethod
def assertDictContains(cls, d1: dict, d2: dict): # noqa
if hasattr(d1, '__dict__'):
d1 = d1.__dict__
if hasattr(d2, '__dict__'):
d2 = d2.__dict__
for k2 in d2.keys():
assert d1, f'dict 1 is empty: {d1}'
assert d2, f'dict 1 is empty: {d2}'
assert k2 in d1, f'dict 1 does not contain {k2}'
if isinstance(d1[k2], dict):
cls.assertDictContains(d1[k2], d2[k2])
else:
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'
assert_dict_contains(d1, d2)
def assertError(self, response, http_status: int, code=None): # noqa
self.assertEqual(response.status_code, http_status)
@ -217,3 +205,19 @@ async def get_all_ws_messages(ws) -> List[dict]:
break
messages.append(json.loads(message))
return messages
def assert_dict_contains(d1: dict, d2: dict):
if hasattr(d1, '__dict__'):
d1 = d1.__dict__
if hasattr(d2, '__dict__'):
d2 = d2.__dict__
for k2 in d2.keys():
assert d1, f'dict 1 is empty: {d1}'
assert d2, f'dict 1 is empty: {d2}'
assert k2 in d1, f'dict 1 does not contain {k2}'
if isinstance(d1[k2], dict):
assert_dict_contains(d1[k2], d2[k2])
else:
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'