diff --git a/alembic/versions/36cae041ad31_files.py b/alembic/versions/36cae041ad31_files.py new file mode 100644 index 00000000..8e69db38 --- /dev/null +++ b/alembic/versions/36cae041ad31_files.py @@ -0,0 +1,53 @@ +"""Files + +Revision ID: 36cae041ad31 +Revises: b97c1313f986 +Create Date: 2022-02-01 11:05:57.605541 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '36cae041ad31' +down_revision = 'b97c1313f986' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('''CREATE TABLE file ( + id BIGSERIAL PRIMARY KEY, + path TEXT NOT NULL, + title TEXT, + mimetype TEXT, + size INTEGER, + modification_datetime TIMESTAMPTZ, + textsearch tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, title), 'A'::"char") + ) stored, + idempotency TEXT + )''') + session.execute('CREATE INDEX file_path_idx ON file(path)') + session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)') + session.execute('CREATE INDEX file_textsearch_idx ON file(textsearch)') + if not DOCKERIZED: + session.execute('ALTER TABLE public.file OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DROP INDEX IF EXISTS file_textsearch_idx') + session.execute('DROP INDEX IF EXISTS file_mimetype_idx') + session.execute('DROP INDEX IF EXISTS file_path_idx') + + session.execute('DROP TABLE IF EXISTS file') diff --git a/app/src/api.js b/app/src/api.js index bcb80594..0cf521b2 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1,4 +1,4 @@ -import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, VIDEOS_API} from "./components/Common"; +import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, secondsToDateTime, VIDEOS_API} from "./components/Common"; import {toast} from 'react-semantic-toasts'; function timeoutPromise(ms, promise) { @@ -303,8 +303,8 @@ export async function deleteArchive(archive_id) { export async function searchArchives(offset, limit, domain = null, searchStr = null) { // Build a search query to retrieve a list of videos from the API - offset = offset || 0; - limit = limit || DEFAULT_LIMIT; + offset = parseInt(offset || 0); + limit = parseInt(limit || DEFAULT_LIMIT); let body = {offset, limit}; if (domain) { body['domain'] = domain; @@ -352,19 +352,16 @@ export async function postDownload(urls, downloader) { return response; } - export async function killDownload(download_id) { let response = await apiPost(`${API_URI}/download/${download_id}/kill`); return response; } - export async function killDownloads() { let response = await apiPost(`${API_URI}/download/kill`); return response; } - export async function startDownloads() { let response = await apiPost(`${API_URI}/download/enable`); return response; @@ -375,10 +372,45 @@ export async function getDownloaders() { return await response.json(); } +const replaceFileDatetimes = (files) => { + for (let i = 0; i < files.length; i++) { + let file = files[i]['modified']; + if (file['modified']) { + files[i]['modified'] = secondsToDateTime(file['modified']); + } + } + return files; +} + +export async function filesSearch(offset, limit, searchStr) { + const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)}; + const response = await apiPost(`${API_URI}/files/search`, body); + + if (response.status === 200) { + let data = await response.json(); + let [files, total] = [data['files'], data['totals']['files']]; + files = replaceFileDatetimes(files); + return [files, total]; + } else { + toast({ + type: 'error', + title: 'Unable to search files', + description: 'Cannot search files. See server logs.', + time: 5000, + }); + } +} + +export async function refreshFiles() { + return await apiPost(`${API_URI}/files/refresh`); +} + export async function getFiles(directories) { let body = {directories: directories || []}; let response = await apiPost(`${API_URI}/files`, body); - return await response.json(); + let {files} = await response.json(); + files = replaceFileDatetimes(files); + return files; } export async function deleteFile(file) { diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 22687488..811c10f2 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -443,9 +443,9 @@ class Downloads extends React.Component { export default function Admin(props) { const links = [ - {text: 'Downloads', to: '/admin', exact: true}, - {text: 'Settings', to: '/admin/settings'}, - {text: 'WROL Mode', to: '/admin/wrol'}, + {text: 'Downloads', to: '/admin', exact: true, key: 'admin'}, + {text: 'Settings', to: '/admin/settings', key: 'settings'}, + {text: 'WROL Mode', to: '/admin/wrol', key: 'wrol'}, ]; return ( diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 78add301..c36de3fb 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -1,10 +1,11 @@ import React, {useState} from "react"; import {Card, Confirm, Container, Icon, Image, Placeholder} from "semantic-ui-react"; -import Paginator, { +import { CardGroupCentered, ClearButton, ExternalLink, PageContainer, + Paginator, SearchInput, TabLinks, textEllipsis, @@ -202,9 +203,8 @@ export function ArchivesList({archives, searchStr, syncArchive, localDeleteArchi } export function Archives() { - const {archivesData, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} = - useArchives(); - const {archives} = archivesData; + const {archives, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} = + useArchives(20); const syncArchive = async (url) => { await postDownload(url, 'archive'); @@ -237,11 +237,7 @@ export function Archives() { setDomain={setDomain} />
- +
) @@ -323,9 +319,9 @@ function Domains() { export function ArchiveRoute(props) { const links = [ - {text: 'Archives', to: '/archive', exact: true}, - {text: 'Domains', to: '/archive/domains'}, - {text: 'Manage', to: '/archive/manage'}, + {text: 'Archives', to: '/archive', exact: true, key: 'archive'}, + {text: 'Domains', to: '/archive/domains', key: 'domains'}, + {text: 'Manage', to: '/archive/manage', key: 'manage'}, ]; return diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 5ee771a6..cd6f2e8a 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -12,7 +12,7 @@ export const ARCHIVES_API = `${API_URI}/archive`; export const OTP_API = `${API_URI}/otp`; export const DEFAULT_LIMIT = 20; -export default class Paginator extends React.Component { +export default class OldPaginator extends React.Component { state = { boundaryRange: 1, siblingRange: 2, @@ -54,6 +54,25 @@ export default class Paginator extends React.Component { } } +export function Paginator({activePage, onPageChange, totalPages, showFirstAndLast}) { + const handlePageChange = (e, {activePage}) => { + onPageChange(activePage); + } + + return ( + + ) +} + export function secondsToDuration(video) { let duration = video.duration; let hours = Math.floor(duration / 3600); @@ -582,6 +601,7 @@ export function TabLinks({links}) { className={'item'} exact={link.exact || false} style={{padding: '1em'}} + key={link.key} > {link.text} diff --git a/app/src/components/Files.js b/app/src/components/Files.js index 149d8329..e40f5949 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -2,8 +2,10 @@ import React from "react"; import FileBrowser from 'react-keyed-file-browser'; import {Button, Confirm, Header, Icon, Modal} from "semantic-ui-react"; import 'react-keyed-file-browser/dist/react-keyed-file-browser.css'; -import {deleteFile, getFiles} from "../api"; -import {humanFileSize, PageContainer, secondsToDateTime} from "./Common"; +import {deleteFile, refreshFiles} from "../api"; +import {Paginator, humanFileSize, PageContainer, SearchInput, TabLinks} from "./Common"; +import {useBrowseFiles, useSearchFiles} from "../hooks/customHooks"; +import {Route} from "react-router-dom"; const icons = { File: , @@ -13,107 +15,142 @@ const icons = { PDF: , }; -class Files extends React.Component { - constructor(props) { - super(props); - this.state = { - deleteOpen: false, - files: [], - openFolders: [], - previewOpen: false, - selectedFile: null, - }; - } +export function Files() { + const {searchFiles, setPage, totalPages, activePage, searchStr, setSearchStr} = useSearchFiles(50); + const {browseFiles, setOpenFolders, fetchFiles} = useBrowseFiles(); - async componentDidMount() { - await this.fetchFiles(); - } + const [deleteOpen, setDeleteOpen] = React.useState(false); + const [previewOpen, setPreviewOpen] = React.useState(false); + const [selectedFile, setSelectedFile] = React.useState(null); - fetchFiles = async () => { - let {files} = await getFiles(this.state.openFolders); - for (let i = 0; i < files.length; i++) { - files[i]['modified'] = secondsToDateTime(files[i]['modified']); - } - this.setState({files: files}); - } - - handleFolderChange = async (file, browserProps) => { + const handleFolderChange = async (file, browserProps) => { let openFolders = Object.keys(browserProps.openFolders); - this.setState({openFolders}, this.fetchFiles); + setOpenFolders(openFolders); } - onDownloadFile = async () => { + const onDownloadFile = async () => { // Open file in new tab/window. - window.open(`/media/${this.state.selectedFile.key}`); + window.open(`/media/${selectedFile.key}`); } - onSelectFile = async (file) => { - this.setState({selectedFile: file}, this.openPreview); + const onSelectFile = async (file) => { + setSelectedFile(file); + openPreview(); } - onDeleteFile = async () => { - await deleteFile(this.state.selectedFile.key); - await this.fetchFiles(); - this.closePreview(); + const onDeleteFile = async () => { + await deleteFile(selectedFile.key); + await fetchFiles(); + closePreview(); } - openPreview = () => { - this.setState({previewOpen: true}); + const openPreview = () => { + setPreviewOpen(true); } - closePreview = () => { - this.setState({previewOpen: false, selectedFile: null, deleteOpen: false}); + const closePreview = () => { + setPreviewOpen(false); + setSelectedFile(null); + setDeleteOpen(false); } - openDelete = () => { - this.setState({deleteOpen: true}); + const openDelete = () => { + setDeleteOpen(true); } - closeDelete = () => { - this.setState({deleteOpen: false}); + const closeDelete = () => { + setDeleteOpen(false); } - render() { - return <> - {this.state.selectedFile && - - File Preview - -
{this.state.selectedFile.name}
-

{humanFileSize(this.state.selectedFile.size)}

- - - - - - - } - <>} // Hide the preview that the 3rd party provided. - /> - + let selectedModal = <>; + if (selectedFile) { + selectedModal = ( + + File Preview + +
{selectedFile.name}
+

{humanFileSize(selectedFile.size)}

+ + + + + + + ); } + + let browser; + let pagination = null; + if (searchStr) { + browser = ( <>} // Hide the preview that the 3rd party provided. + />); + pagination = ( +
+ +
); + } else { + browser = ( <>} // Hide the preview that the 3rd party provided. + />); + } + + return <> + + {selectedModal} + {browser} + {pagination} + +} + +function ManageFiles() { + return (<> + + + ) } export function FilesRoute(props) { + const links = [ + {text: 'Files', to: '/files', exact: true, key: 'files'}, + {text: 'Manage', to: '/files/manage', exact: true, key: 'manage'}, + ]; + return ( - + + + ); } diff --git a/app/src/components/Inventory.js b/app/src/components/Inventory.js index a9039744..b87501e3 100644 --- a/app/src/components/Inventory.js +++ b/app/src/components/Inventory.js @@ -1090,8 +1090,8 @@ class InventoryAddList extends React.Component { export function InventoryRoute(props) { const links = [ - {text: 'List', to: '/inventory', exact: true}, - {text: 'Summary', to: '/inventory/summary'}, + {text: 'List', to: '/inventory', exact: true, key: 'inventory'}, + {text: 'Summary', to: '/inventory/summary', key: 'summary'}, ]; return ( diff --git a/app/src/components/Nav.js b/app/src/components/Nav.js index 18e00ea4..1725e888 100644 --- a/app/src/components/Nav.js +++ b/app/src/components/Nav.js @@ -5,17 +5,17 @@ import {Dropdown, Menu, Responsive} from "semantic-ui-react"; const responsiveWidth = 500; const links = [ - {text: 'Videos', to: '/videos'}, - {text: 'Inventory', to: '/inventory'}, - {text: 'Archive', to: '/archive'}, - {text: 'Files', to: '/files'}, + {text: 'Videos', to: '/videos', key: 'videos'}, + {text: 'Inventory', to: '/inventory', key: 'inventory'}, + {text: 'Archive', to: '/archive', key: 'archive'}, + {text: 'Files', to: '/files', key: 'files'}, { - text: 'Apps', links: [ + key: 'apps', text: 'Apps', links: [ {to: '/apps/otp', text: 'One Time Pad', exact: true}, ] }, ]; -const admin = {to: '/admin', text: 'Admin', exact: true}; +const admin = {to: '/admin', text: 'Admin', exact: true, key: 'admin'}; const rightLinks = [admin,]; const collapsedLinks = links.concat([admin,]); @@ -67,22 +67,20 @@ export function NavBar() { {/*Show the links in a menu when on desktop*/} {links.map((link) => { return ( - + ) })} - {rightLinks.map((link) => { - return () - })} + {rightLinks.map((link) => )} {/*Show the menu items in a dropdown when on mobile*/} - {collapsedLinks.map((link) => { - return () - })} + {collapsedLinks.map((link) => + + )} diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 06bf309b..f068f252 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -1,6 +1,6 @@ import React from 'react'; import {Link, Route} from "react-router-dom"; -import Paginator, { +import OldPaginator, { changePageHistory, DEFAULT_LIMIT, defaultSearchOrder, @@ -401,7 +401,7 @@ class Videos extends React.Component { if (totalPages) { pagination = (
- { @@ -68,15 +68,26 @@ export const useDomains = () => { return [domains, total]; } -export const useArchives = (defaultLimit = 20) => { - const [archivesData, setArchives] = useState({archives: null, total: 0}); - const [totalPages, setTotalPages] = useState(0); - const [activePage, setActivePage] = useState(1); - - let [offset, setOffset] = useSearchParam('o'); +export const useArchives = ({defaultLimit = 20}) => { let [limit, setLimit] = useSearchParam('l', defaultLimit); - let [domain, setDomain] = useSearchParam('domain'); + let [offset, setOffset] = useSearchParam('o'); let [searchStr, setSearchStr] = useSearchParam('q'); + let [domain, setDomain] = useSearchParam('domain'); + + const [archives, setArchives] = useState(null); + const [totalPages, setTotalPages] = useState(0); + const [activePage, setActivePage] = useState((offset / limit) + 1); + + const search = async () => { + setArchives(null); + const [archives, total] = await searchArchives(offset, limit, domain, searchStr); + setTotalPages(Math.floor(total / limit) + 1); + setArchives(archives); + } + + useEffect(() => { + search(); + }, [offset, limit, domain, searchStr]); const setPage = (i) => { i = parseInt(i); @@ -85,20 +96,8 @@ export const useArchives = (defaultLimit = 20) => { setActivePage(i); } - const search = async (term) => { - setArchives({archives: null, total: 0}); - setTotalPages(0); - const [archives, total] = await searchArchives(offset, limit, domain, term); - setTotalPages(Math.floor(total / limit) + 1); - setArchives({archives, total}); - } - - useEffect(() => { - search(searchStr); - }, [offset, limit, domain, searchStr]); - return { - archivesData, + archives, totalPages, setTotalPages, offset, @@ -130,3 +129,52 @@ export const useVersion = () => { return version; } + +export const useSearchFiles = ({defaultLimit = 50}) => { + let [limit, setLimit] = useSearchParam('l', defaultLimit); + let [offset, setOffset] = useSearchParam('o'); + let [searchStr, setSearchStr] = useSearchParam('q'); + + const [searchFiles, setSearchFiles] = useState([]); + const [totalPages, setTotalPages] = useState(0); + const [activePage, setActivePage] = useState(1); + + const localSearchFiles = async () => { + setSearchFiles([]); + setTotalPages(0); + if (searchStr) { + let [files, total] = await filesSearch(offset, limit, searchStr); + setSearchFiles(files); + setTotalPages(Math.floor(total / limit) + 1); + } + } + + useEffect(() => { + localSearchFiles(); + }, [searchStr, limit, offset, activePage]); + + const setPage = (i) => { + i = parseInt(i); + let l = parseInt(limit); + setOffset((l * i) - l); + setActivePage(i); + } + + return {searchFiles, totalPages, limit, setLimit, setOffset, searchStr, setSearchStr, activePage, setPage}; +} + +export const useBrowseFiles = () => { + const [browseFiles, setBrowseFiles] = useState([]); + const [openFolders, setOpenFolders] = useState([]); + + const fetchFiles = async () => { + const files = await getFiles(openFolders); + setBrowseFiles(files); + } + + useEffect(() => { + fetchFiles(); + }, [openFolders]) + + return {browseFiles, openFolders, setOpenFolders, fetchFiles}; +} diff --git a/modules/files/api.py b/modules/files/api.py index daa296bc..45c78f92 100644 --- a/modules/files/api.py +++ b/modules/files/api.py @@ -6,7 +6,7 @@ from sanic import response from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi.common import get_media_directory +from wrolpi.common import get_media_directory, api_param_limiter from wrolpi.errors import InvalidFile from wrolpi.root_api import get_blueprint, json_response from . import lib, schema @@ -61,3 +61,21 @@ async def delete_file(_: Request, body: schema.DeleteRequest): raise InvalidFile('file cannot be empty') lib.delete_file(body.file) return response.empty() + + +files_limit_limiter = api_param_limiter(100) +files_offset_limiter = api_param_limiter(100, 0) + + +@bp.post('/refresh') +@openapi.description('Find and index all files') +async def refresh(_: Request): + lib.refresh_files() + return response.empty() + + +@bp.post('/search') +@validate(schema.FilesSearchRequest) +async def search_files(_: Request, body: schema.FilesSearchRequest): + files, total = lib.search(body.search_str, body.limit, body.offset) + return json_response(dict(files=files, totals=dict(files=total))) diff --git a/modules/files/lib.py b/modules/files/lib.py index 6224be4e..ac5d3c80 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -1,8 +1,24 @@ +import asyncio +import re +from functools import wraps from pathlib import Path from typing import List +from uuid import uuid4 -from wrolpi.common import get_media_directory, wrol_mode_check +import magic +import psycopg2 +from sqlalchemy.orm import Session + +from modules.files.models import File +from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks, logger +from wrolpi.dates import from_timestamp +from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidFile +from wrolpi.vars import PYTEST + +logger = logger.getChild(__name__) + +__all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search'] def filter_parent_directories(directories: List[Path]) -> List[Path]: @@ -56,3 +72,101 @@ def delete_file(file: str): if file.is_dir() or not file.is_file(): raise InvalidFile(f'Invalid file {file}') file.unlink() + + +FILE_NAME_REGEX = re.compile(r'[_ ]') + + +def split_file_name(path: Path) -> List[str]: + """ + Split a file name into words. + + >>> split_file_name(Path('foo.mp4')) + ['foo'] + >>> split_file_name(Path('foo bar.mp4')) + ['foo', 'bar'] + """ + words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)] + return words + + +def upsert_file(path: Path, session: Session) -> File: + """Update/insert a File in the DB. Gather metadata about it.""" + file = session.query(File).filter_by(path=path).one_or_none() + if not file: + file = File(path=path) + session.add(file) + if not file.mimetype: + file.mimetype = magic.from_file(str(path), mime=True) + if not file.title: + file.title = split_file_name(path) + if not file.size or not file.modification_datetime: + stat = path.stat() + file.size = stat.st_size + file.modification_datetime = file.modification_datetime or from_timestamp(stat.st_mtime) + + return file + + +def _refresh_files(): + """Find and index all files""" + logger.info('Refreshing Files') + # Mark all files as unverified. Any records that are unverified after refresh will be deleted. + with get_db_curs(commit=True) as curs: + curs.execute('UPDATE file SET idempotency=null') + + paths = filter(lambda i: i.is_file(), walk(get_media_directory())) + idempotency = str(uuid4()) + for chunk in chunks(paths, 20): + with get_db_session(commit=True) as session: + for path in chunk: + file = upsert_file(path, session) + file.idempotency = idempotency + + # Remove any records where the file no longer exists. + with get_db_curs(commit=True) as curs: + curs.execute('DELETE FROM file WHERE idempotency IS NULL') + + logger.info('Done refreshing Files') + + +@wraps(_refresh_files) +def refresh_files(): + """Schedule a refresh task if not testing. If testing, do a synchronous refresh.""" + if PYTEST: + return _refresh_files() + + async def _(): + return _refresh_files() + + asyncio.create_task(_()) + + +def search(search_str: str, limit: int, offset: int): + """ + Search the "title" of each file. The title is the words in the file's stem. + """ + with get_db_curs() as curs: + stmt = ''' + SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total + FROM file + WHERE textsearch @@ websearch_to_tsquery(%(search_str)s) + OFFSET %(offset)s LIMIT %(limit)s + ''' + params = dict(search_str=search_str, offset=offset, limit=limit) + curs.execute(stmt, params) + + try: + results = list(map(dict, curs.fetchall())) + except psycopg2.ProgrammingError: + # No files. + return [], 0 + + total = results[0]['total'] if results else 0 + ranked_ids = [i['id'] for i in results] + + with get_db_session() as session: + results = get_ranked_models(ranked_ids, File, session) + results = [i.__json__() for i in results] + + return results, total diff --git a/modules/files/models.py b/modules/files/models.py new file mode 100644 index 00000000..09ee70c8 --- /dev/null +++ b/modules/files/models.py @@ -0,0 +1,36 @@ +from sqlalchemy import Integer, Column, String, Computed +from sqlalchemy.orm import deferred + +from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector +from wrolpi.dates import TZDateTime +from wrolpi.media_path import MediaPathType + + +class File(ModelHelper, Base): + __tablename__ = 'file' + id = Column(Integer, primary_key=True) + path = Column(MediaPathType) + + idempotency = Column(String) + mimetype = Column(String) + modification_datetime = Column(TZDateTime) + size = Column(Integer) + title = deferred(Column(String)) # used by textsearch column + + textsearch = deferred( + Column(tsvector, Computed('''to_tsvector('english'::regconfig, title)'''))) + + def __repr__(self): + return f'' + + def __json__(self): + path = self.path.path.relative_to(get_media_directory()) + d = dict( + id=self.id, + mimetype=self.mimetype, + modified=self.modification_datetime, # React browser expects this. + path=path, + size=self.size, + key=path, + ) + return d diff --git a/modules/files/schema.py b/modules/files/schema.py index 8952834c..af2bf73b 100644 --- a/modules/files/schema.py +++ b/modules/files/schema.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import List +from typing import List, Optional @dataclass @@ -10,3 +10,10 @@ class FilesRequest: @dataclass class DeleteRequest: file: str + + +@dataclass +class FilesSearchRequest: + search_str: str + limit: Optional[int] = 20 + offset: Optional[int] = 0 diff --git a/modules/files/test/test_api.py b/modules/files/test/test_api.py index 0fb1ea41..3931282b 100644 --- a/modules/files/test/test_api.py +++ b/modules/files/test/test_api.py @@ -6,6 +6,8 @@ from unittest import mock import pytest from wrolpi.errors import API_ERRORS, WROLModeEnabled +from wrolpi.test.common import assert_dict_contains +from wrolpi.vars import PROJECT_DIR def test_list_files_api(test_client, make_files_structure, test_directory): @@ -114,3 +116,44 @@ def test_delete_wrol_mode(test_client): request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'})) assert response.status_code == HTTPStatus.FORBIDDEN assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + +def do_search(test_client, search_str, total, expected): + content = json.dumps({'search_str': search_str}) + request, response = test_client.post('/api/files/search', content=content) + assert response.json['totals']['files'] == total + for file, expected in zip_longest(response.json['files'], expected): + assert_dict_contains(file, expected) + # FileBrowser in React requires a key. We use the path. + assert file['path'] == file['key'] + + +def test_files_search(test_session, test_client, make_files_structure): + # You can search an empty directory. + do_search(test_client, 'nothing', 0, []) + + # Create files in the temporary directory. Add some contents so the mimetype can be tested. + files = [ + 'foo_is_the_name.txt', + 'archives/bar.txt', + 'baz.mp4', + 'baz two.mp4' + ] + foo, bar, baz, baz2 = make_files_structure(files) + foo.write_text('foo contents') + bar.write_text('the bar contents') + baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()) + baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()) + + # Refresh so files can be searched. + request, response = test_client.post('/api/files/refresh') + assert response.status_code == HTTPStatus.NO_CONTENT + + do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)]) + do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)]) + do_search(test_client, 'baz', 2, [ + dict(path='baz two.mp4', mimetype='video/mp4', size=1055736), + dict(path='baz.mp4', mimetype='video/mp4', size=1055736), + ]) + do_search(test_client, 'two', 1, [dict(path='baz two.mp4', mimetype='video/mp4', size=1055736)]) + do_search(test_client, 'nothing', 0, []) diff --git a/modules/files/test/test_lib.py b/modules/files/test/test_lib.py index 1a53bb29..a15862d5 100644 --- a/modules/files/test/test_lib.py +++ b/modules/files/test/test_lib.py @@ -1,9 +1,12 @@ from pathlib import Path +from typing import List, Iterable import pytest from modules.files import lib +from modules.files.models import File from wrolpi.errors import InvalidFile +from wrolpi.media_path import MediaPathType def test_list_files(make_files_structure, test_directory): @@ -122,3 +125,47 @@ def test_delete_file(make_files_structure, test_directory): lib.delete_file('does not exist') assert (test_directory / 'baz').is_dir() + + +@pytest.mark.parametrize( + 'path,expected', + [ + ('foo.mp4', ['foo']), + ('foo bar.txt', ['foo', 'bar']), + ('foo_bar.mp4', ['foo', 'bar']), + ('foo-bar', ['foo-bar']), + ('123 foo bar', ['123', 'foo', 'bar']), + ('123foo bar', ['123foo', 'bar']), + ] +) +def test_split_file_name(path, expected): + assert lib.split_file_name(Path(path)) == expected + + +def test_refresh_files(test_session, make_files_structure, test_directory): + files = make_files_structure([ + 'foo.txt', + 'bar.txt', + 'baz.txt', + ]) + foo, bar, baz = files + + def get_relative_strs(files_: Iterable[MediaPathType]) -> List[str]: + return sorted([str(i.path.path.relative_to(test_directory)) for i in files_]) + + lib.refresh_files() + results = test_session.query(File) + assert get_relative_strs(results) == ['bar.txt', 'baz.txt', 'foo.txt'] + + baz.unlink() + + lib.refresh_files() + results = test_session.query(File) + assert get_relative_strs(results) == ['bar.txt', 'foo.txt'] + + foo.unlink() + bar.unlink() + + lib.refresh_files() + results = test_session.query(File) + assert get_relative_strs(results) == [] diff --git a/modules/videos/captions.py b/modules/videos/captions.py index 5b60d83b..8f098c1a 100755 --- a/modules/videos/captions.py +++ b/modules/videos/captions.py @@ -1,5 +1,4 @@ #! /usr/bin/env python3 -import pathlib from pathlib import Path from typing import Generator, Union @@ -7,14 +6,12 @@ import srt import webvtt from wrolpi.common import logger -from wrolpi.media_path import MediaPath -from .models import Video + +__all__ = ['get_captions'] def get_caption_text(caption_path: Union[str, Path]) -> Generator: - """ - Return all text from each caption of a caption file. - """ + """Return all text from each caption of a caption file.""" if str(caption_path).endswith('vtt'): for caption in webvtt.read(caption_path): text = str(caption.text).strip() @@ -38,15 +35,8 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator: yield line -def get_video_captions(video: Video): - """ - Parse video captions from the video's captions file. - """ - if isinstance(video.caption_path, MediaPath): - caption_path = video.caption_path.path - else: - caption_path = pathlib.Path(video.caption_path) - +def get_captions(caption_path: Path): + """Parse video captions from the video's captions file.""" try: lines = get_unique_caption_lines(str(caption_path)) block = '\n'.join(lines) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index f6610a84..f6cb4fab 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -14,7 +14,7 @@ from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST -from .captions import get_video_captions +from .captions import get_captions from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \ get_video_duration, is_valid_poster, convert_image, generate_video_poster from .models import Channel, Video @@ -188,7 +188,7 @@ def validate_video(video: Video, channel_generate_poster: bool): # Video duration was not in the info json, use ffprobe. video.duration = get_video_duration(video_path) if not video.caption and video.caption_path: - video.caption = get_video_captions(video) + video.caption = get_captions(video.caption_path.path) if not video.size or not video.modification_datetime: stat = video_path.stat() diff --git a/modules/videos/test/test_captions.py b/modules/videos/test/test_captions.py index 894e80d1..3dbc4cdb 100644 --- a/modules/videos/test/test_captions.py +++ b/modules/videos/test/test_captions.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path from modules.videos import captions @@ -46,11 +47,11 @@ class TestCaption(TestAPI): video1 = Video(title='scream', caption_path=str(self.vtt_path1)) session.add(video1) session.commit() - video1.caption = captions.get_video_captions(video1) + video1.caption = captions.get_captions(video1.caption_path.path) video2 = Video(title='bar', caption_path=str(self.vtt_path2)) session.add(video2) session.commit() - video2.caption = captions.get_video_captions(video2) + video2.caption = captions.get_captions(video2.caption_path.path) session.commit() session.flush() @@ -91,3 +92,13 @@ class TestCaption(TestAPI): # Matches video2.title select_textsearch('bar') self.assertEqual(curs.fetchall(), [[2, ]]) + + +def test_bad_caption(test_directory): + """A caption file that cannot be parsed is ignored.""" + bad_caption_file = PROJECT_DIR / 'test/bad_caption.en.vtt' + test_file = test_directory / 'bad_caption.en.vtt' + shutil.copy(bad_caption_file, test_file) + + result = captions.get_captions(test_file) + assert result is None diff --git a/requirements.txt b/requirements.txt index 9e681a9d..e5954ff3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ Pint==0.18 psycopg2-binary==2.9.3 pytest-asyncio==0.17.2 pytest==6.2.5 +python-magic==0.4.24 pytz==2021.3 PyYAML==6.0 requests==2.27.1 @@ -15,7 +16,7 @@ sanic-testing==0.8.2 sanic==21.12.1 selenium==4.1.0 SQLAlchemy==1.3.22 -srt==3.5.0 +srt==3.5.1 websockets==10.1 webvtt-py==0.4.6 yt-dlp==2022.1.21 diff --git a/test/bad_caption.en.vtt b/test/bad_caption.en.vtt new file mode 100644 index 00000000..463b74ef --- /dev/null +++ b/test/bad_caption.en.vtt @@ -0,0 +1,8 @@ +WEBVTT +Kind: captions +Language: en + +00:00:00.070 --> 00:00:06.380 +this caption is bad + +00:05:08.650 --> 00:05:11.26 diff --git a/wrolpi/common.py b/wrolpi/common.py index 19134a0a..9056e473 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -638,3 +638,11 @@ def zig_zag(low: ZIG_TYPE, high: ZIG_TYPE) -> Generator[ZIG_TYPE, None, None]: if num >= high: divisor *= 2 num = low + (diff / divisor) + + +def walk(path: Path) -> Generator[Path, None, None]: + """Walk a directory structure yielding all files and directories.""" + for path in path.iterdir(): + yield path + if path.is_dir(): + yield from walk(path) diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py index a93bc3dd..efbdd4a5 100644 --- a/wrolpi/test/common.py +++ b/wrolpi/test/common.py @@ -76,19 +76,7 @@ class PytestCase: @classmethod def assertDictContains(cls, d1: dict, d2: dict): # noqa - if hasattr(d1, '__dict__'): - d1 = d1.__dict__ - if hasattr(d2, '__dict__'): - d2 = d2.__dict__ - - for k2 in d2.keys(): - assert d1, f'dict 1 is empty: {d1}' - assert d2, f'dict 1 is empty: {d2}' - assert k2 in d1, f'dict 1 does not contain {k2}' - if isinstance(d1[k2], dict): - cls.assertDictContains(d1[k2], d2[k2]) - else: - assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1' + assert_dict_contains(d1, d2) def assertError(self, response, http_status: int, code=None): # noqa self.assertEqual(response.status_code, http_status) @@ -217,3 +205,19 @@ async def get_all_ws_messages(ws) -> List[dict]: break messages.append(json.loads(message)) return messages + + +def assert_dict_contains(d1: dict, d2: dict): + if hasattr(d1, '__dict__'): + d1 = d1.__dict__ + if hasattr(d2, '__dict__'): + d2 = d2.__dict__ + + for k2 in d2.keys(): + assert d1, f'dict 1 is empty: {d1}' + assert d2, f'dict 1 is empty: {d2}' + assert k2 in d1, f'dict 1 does not contain {k2}' + if isinstance(d1[k2], dict): + assert_dict_contains(d1[k2], d2[k2]) + else: + assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'