rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Creating Video.ffprobe_json column which stores ffprobe information about the video file. Creating find_corrupt_videos.py script which automatically re-downloads videos which do not have both video and audio streams.

Browse source
This commit is contained in:
Roland 2023-07-27 15:53:47 -06:00
parent 097c3143e1
commit 36083200e6
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
39 changed files with 598 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

35
alembic/versions/8bdb48a486b9_video_ffprobe_json.py Normal file
View file

@ -0,0 +1,35 @@
"""Video ffprobe json.
Revision ID: 8bdb48a486b9
Revises: bf704baa185d
Create Date: 2023-07-24 10:59:17.426100
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '8bdb48a486b9'
down_revision = 'bf704baa185d'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video ADD COLUMN ffprobe_json JSON')
# Re-index all videos so the ffprobe data is fetched.
session.execute("UPDATE file_group SET indexed=false WHERE mimetype like 'video/%'")
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS ffprobe_json')

4
app/src/Events.js
View file

@ -16,6 +16,8 @@ function handleEvents(events) {
const {event, message, subject, dt, url} = e;
document.dispatchEvent(new CustomEvent(apiEventName, {detail: e}));
console.debug(`Got event: ${event}`);
if (event === 'directory_refresh') {
eventToast(
'Refresh started',
@ -26,7 +28,7 @@ function handleEvents(events) {
if (subject && newestEvents[subject] && newestEvents[subject] > dt) {
console.debug(`Already handled newer event of "${subject}".`);
return;
} else if (event === 'global_refresh_completed') {
} else if (event === 'refresh_completed') {
eventToast('Refresh completed', 'All files have been refreshed.');
}

26
app/src/api.js
View file

@ -539,23 +539,17 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
}
}
export async function refreshFiles() {
return await apiPost(`${API_URI}/files/refresh`);
}
export async function refreshDirectoryFiles(paths) {
let body = {paths: paths};
try {
return await apiPost(`${API_URI}/files/refresh`, body);
} catch (e) {
console.error(e);
toast({
type: 'error',
title: 'Unable to refresh directory',
description: 'Cannot refresh the files in the directory. See server logs.',
time: 5000,
});
export async function refreshFiles(paths) {
let response;
if (paths && paths.constructor === Array) {
const body = {paths};
console.debug(`Refreshing: ${paths}`);
response = await apiPost(`${API_URI}/files/refresh`, body);
} else {
console.debug(`Refreshing all files`);
response = await apiPost(`${API_URI}/files/refresh`);
}
return response;
}
export async function makeDirectory(path) {

28
app/src/components/Archive.js
View file

@ -131,11 +131,11 @@ function ArchivePage() {
historyList = <FileCards files={history}/>;
}
const domain = data.domain ? data.domain.domain : null;
const domain = data.domain ? data.domain : null;
let domainHeader;
if (domain) {
const domainUrl = `/archive?domain=${domain}`;
domainHeader = <Header as='h5'>
domainHeader = <Header as='h4'>
<CardLink to={domainUrl}>
{domain}
</CardLink>
@ -272,7 +272,7 @@ export function DomainsPage() {
filteredDomains = domains.filter(i => re.test(i['domain']));
}
const row = ({domain, url_count}) => {
const domainRow = ({domain, url_count, size}) => {
return <TableRow key={domain}>
<TableCell>
<Link to={`/archive?domain=${domain}`}>
@ -280,12 +280,14 @@ export function DomainsPage() {
</Link>
</TableCell>
<TableCell>{url_count}</TableCell>
<TableCell>{humanFileSize(size)}</TableCell>
</TableRow>
}
const headers = [
{key: 'domain', text: 'Domain', sortBy: 'domain', width: 12},
{key: 'archives', text: 'Archives', sortBy: 'url_count', width: 2},
{key: 'Size', text: 'Size', sortBy: 'size', width: 2},
];
return <>
@ -298,7 +300,7 @@ export function DomainsPage() {
<SortableTable
tableProps={{unstackable: true}}
data={filteredDomains}
rowFunc={(i, sortData) => row(i)}
rowFunc={(i, sortData) => domainRow(i)}
rowKey='domain'
tableHeaders={headers}
/>
@ -362,11 +364,10 @@ function ArchivesPage() {
}
}
const onDelete = async (e) => {
e.preventDefault();
const onDelete = async () => {
setDeleteOpen(false);
const archiveIds = archives.filter(i => selectedArchives.indexOf(i['primary_path']) >= 0)
.map(i => i['archive']['id']);
.map(i => i['data']['id']);
await deleteArchives(archiveIds);
await fetchArchives();
setSelectedArchives([]);
@ -383,18 +384,13 @@ function ArchivesPage() {
}
const selectElm = <div style={{marginTop: '0.5em'}}>
<Button
<APIButton
color='red'
disabled={_.isEmpty(selectedArchives)}
onClick={() => setDeleteOpen(true)}
>Delete</Button>
<Confirm
open={deleteOpen}
content='Are you sure you want to delete these archives files? This cannot be undone.'
confirmButton='Delete'
onCancel={() => setDeleteOpen(false)}
onConfirm={onDelete}
/>
confirmContent='Are you sure you want to delete these archives files? This cannot be undone.'
onClick={onDelete}
>Delete</APIButton>
<Button
color='grey'
onClick={invertSelection}

24
app/src/components/Common.js
View file

@ -541,6 +541,7 @@ export function CardPoster({to, file}) {
const {s} = useContext(ThemeContext);
// Used to center posters in CardIcon.
const style = {display: 'flex', justifyContent: 'center', ...s['style']};
const navigate = useNavigate();
const cardTagIcon = <div className="ui green left corner label">
<i aria-hidden="true" className="tag icon"></i>
@ -549,8 +550,27 @@ export function CardPoster({to, file}) {
let posterPath = findPosterPath(file);
if (!posterPath) {
// No poster, use a FileIcon.
if (!posterPath && to && (to.startsWith('/download/') || to.startsWith('/media/'))) {
// No poster, link is to download a file directly.
return <ExternalCardLink to={to}>
<CardIcon onClick={() => {
window.location.href = to;
return null
}}>
{imageLabel}
<FileIcon file={file}/>
</CardIcon>
</ExternalCardLink>
} else if (!posterPath && to) {
// No poster, link to the full page.
return <Link to={to}>
<CardIcon onClick={() => navigate(to)}>
{imageLabel}
<FileIcon file={file}/>
</CardIcon>
</Link>
} else if (!posterPath) {
// No poster, no to, use a Preview.
return <PreviewLink file={file}>
<CardIcon>
{imageLabel}

16
app/src/components/DonatePage.js
View file

@ -61,6 +61,22 @@ export function DonatePage() {
<Segment>
<Header as='h1'>Donate</Header>
<Header as='h3'>We appreciate any support you can provide to WROLPi!</Header>
<p>
<Button
size='huge'
icon='paypal'
as='a'
color='blue'
href='https://www.paypal.com/donate/?hosted_button_id=ZH2CN92SMJ66N'
label={{
basic: true,
content: 'Donate on PayPal',
color: 'blue'
}}
/>
</p>
<CoinRow header='Monero' qrCodeValue={moneroContent} address={moneroAddress} buttonColor='orange'/>
<CoinRow header='Litecoin' qrCodeValue={litecoinContent} address={litecoinAddress} buttonColor='grey'/>
<CoinRow header='Ethereum' qrCodeValue={ethereumContent} address={ethereumAddress} buttonColor='blue'/>

49
app/src/components/Files.js
View file

@ -37,7 +37,7 @@ import {
useQuery,
useSearchFiles,
useSearchFilter,
useSearchView,
useSearchView, useStatusFlag,
useWROLMode
} from "../hooks/customHooks";
import {Route, Routes} from "react-router-dom";
@ -62,7 +62,7 @@ import {SelectableTable} from "./Tables";
import {VideoCard, VideoRowCells} from "./Videos";
import _ from 'lodash';
import {FileBrowser} from "./FileBrowser";
import {refreshDirectoryFiles, refreshFiles} from "../api";
import {refreshFiles} from "../api";
import {useSubscribeEventName} from "../Events";
import {TagsSelector} from "../Tags";
import {Headlines} from "./Headline";
@ -571,26 +571,37 @@ export function FilesRoute() {
</PageContainer>;
}
export function FilesRefreshButton() {
const {status} = useContext(StatusContext);
const useRefresh = () => {
const refreshing = useStatusFlag('refreshing');
const refreshingDirectory = useStatusFlag('refreshing_directory');
const wrolModeEnabled = useWROLMode();
const refreshing = status && status['flags'] && status['flags'].indexOf('refreshing') >= 0;
const refreshingDirectory = status && status['flags'] && status['flags'].indexOf('refreshing_directory') >= 0;
const [loading, setLoading] = React.useState(false);
const handleClick = async () => {
const localRefreshFiles = async (paths) => {
setLoading(true);
await refreshFiles();
await refreshFiles(paths);
}
// Clear loading when global refresh event completes.
useSubscribeEventName('global_refresh_completed', () => setLoading(false));
useSubscribeEventName('refresh_completed', () => setLoading(false));
return {
refreshing,
refreshingDirectory,
wrolModeEnabled,
loading,
refreshFiles: localRefreshFiles,
}
}
export function FilesRefreshButton() {
const {refreshing, refreshingDirectory, wrolModeEnabled, loading, refreshFiles} = useRefresh();
return <Button icon
labelPosition='left'
loading={loading || refreshing || refreshingDirectory}
onClick={handleClick}
onClick={() => refreshFiles()}
disabled={wrolModeEnabled || loading || refreshing}>
<Icon name='refresh'/>
Refresh All
@ -598,26 +609,12 @@ export function FilesRefreshButton() {
}
export function DirectoryRefreshButton({paths}) {
const {status} = useContext(StatusContext);
const refreshing = status && status['flags'] && status['flags'].indexOf('refreshing') >= 0;
const refreshingDirectory = status && status['flags'] && status['flags'].indexOf('refreshing_directory') >= 0;
const wrolModeEnabled = useWROLMode();
const [loading, setLoading] = React.useState(false);
const handleClick = async () => {
setLoading(true);
await refreshDirectoryFiles(paths);
}
// Clear loading when global refresh event completes.
useSubscribeEventName('global_refresh_completed', () => setLoading(false));
useSubscribeEventName('directory_refresh_completed', () => setLoading(false));
const {refreshing, refreshingDirectory, wrolModeEnabled, loading, refreshFiles} = useRefresh();
return <Button icon
labelPosition='left'
loading={loading || refreshing || refreshingDirectory}
onClick={handleClick}
onClick={() => refreshFiles(paths)}
disabled={wrolModeEnabled || loading || refreshing || refreshingDirectory}
>
<Icon name='refresh'/>

15
app/src/components/VideoPlayer.js
View file

@ -21,6 +21,7 @@ import {ThemeContext} from "../contexts/contexts";
import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme";
import {VideoCard} from "./Videos";
import {TagsSelector} from "../Tags";
import {Label} from "semantic-ui-react";
const MEDIA_PATH = '/media';
@ -109,6 +110,9 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
<h3>Censored</h3>
<p>{video.censored ? 'Yes' : 'No'}</p>
<h3>Codec Names</h3>
<>{video.codec_names ? video.codec_names.map(i => <Label>{i}</Label>) : 'N/A'}</>
</TabPane>
}
@ -133,7 +137,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
<h4>Caption Files</h4>
{caption_files &&
caption_files.map(i => <PreviewPath key={i['path']} {...i}>{i['path']}</PreviewPath>)
caption_files.map(i => <p key={i['path']}><PreviewPath {...i}>{i['path']}</PreviewPath></p>)
}
<h4>Poster File</h4>
@ -161,6 +165,13 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
await fetchVideo();
}
let videoSource = <source src={videoUrl}/>;
if (video.codec_names && video.codec_names.indexOf('mp4') >= 0) {
videoSource = <source src={videoUrl} type="video/mp4"/>
} else if (video.codec_names && video.codec_names.indexOf('vp9') >= 0) {
videoSource = <source src={videoUrl} type='video/mp4;codecs="vp9, vorbis"'/>
}
return <>
<Container style={{margin: '1em'}}>
<BackButton/>
@ -174,7 +185,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
style={{maxWidth: '100%'}}
ref={videoRef}
>
<source src={videoUrl} type="video/mp4"/>
{videoSource}
{/* Only WebVTT captions can be displayed. */}
{captionUrls.map(i => <track key={i} kind="captions" label="English" src={i} srcLang="en" default/>)}
</video>

44
app/src/components/Videos.js
View file

@ -1,6 +1,7 @@
import React, {useContext, useEffect, useState} from 'react';
import {Link, Route, Routes, useParams} from "react-router-dom";
import {
APIButton,
CardLink,
CardPoster,
cardTitleWrapper,
@ -13,6 +14,7 @@ import {
isoDatetimeToString,
mimetypeColor,
PageContainer,
PreviewLink,
scrollToTop,
SearchInput,
SortButton,
@ -25,7 +27,6 @@ import {
CardContent,
CardDescription,
CardHeader,
Confirm,
Container,
Image,
PlaceholderHeader,
@ -60,7 +61,6 @@ function VideosPage() {
const {channelId} = useParams();
const {searchParams} = useQuery();
const [selectedVideos, setSelectedVideos] = useState([]);
const [deleteOpen, setDeleteOpen] = useState(false);
let searchOrder = defaultVideoOrder;
if (searchParams.get('order')) {
@ -118,9 +118,7 @@ function VideosPage() {
setSelectedVideos(newSelectedVideos);
}
const onDelete = async (e) => {
e.preventDefault();
setDeleteOpen(false);
const onDelete = async () => {
const videoIds = videos.filter(i => selectedVideos.indexOf(i['primary_path']) >= 0).map(i => i['video']['id']);
await deleteVideos(videoIds);
await fetchVideos();
@ -138,18 +136,13 @@ function VideosPage() {
}
const selectElm = <div style={{marginTop: '0.5em'}}>
<Button
<APIButton
color='red'
disabled={_.isEmpty(selectedVideos)}
onClick={() => setDeleteOpen(true)}
>Delete</Button>
<Confirm
open={deleteOpen}
content='Are you sure you want to delete these video files? This cannot be undone.'
confirmButton='Delete'
onCancel={() => setDeleteOpen(false)}
onConfirm={onDelete}
/>
confirmContent='Are you sure you want to delete these video files? This cannot be undone.'
onClick={onDelete}
>Delete</APIButton>
<Button
color='grey'
onClick={() => invertSelection()}
@ -290,18 +283,35 @@ export function VideoCard({file}) {
const {video} = file;
const {s} = useContext(ThemeContext);
let video_url = `/videos/video/${video.id}`;
// Default to video FilePreview for lone video files.
let video_url;
const upload_date = isoDatetimeToString(video.upload_date);
// A video may not have a channel.
const channel = video.channel ? video.channel : null;
let channel_url = null;
if (channel) {
// Link to Video in the Channel if possible.
channel_url = `/videos/channel/${channel.id}/video`;
video_url = `/videos/channel/${channel.id}/video/${video.id}`;
} else if (file.files.length > 1) {
// No Channel, but the video has multiple files (subtitles, etc.). Use the full VideoPage.
video_url = `/videos/video/${video.id}`
}
let poster = <CardPoster to={video_url} file={file}/>;
let header = <p {...s}>{cardTitleWrapper(file.title || file.name || video.stem || video.video_path)}</p>;
if (video_url) {
// Link to Channel-Video page or Video page.
header = <Link to={video_url} className="no-link-underscore card-link">{header}</Link>;
} else {
// Video is just a lone video file.
header = <PreviewLink file={file}>
{header}
</PreviewLink>;
}
const color = mimetypeColor(file.mimetype);
return <Card color={color}>
{poster}
@ -309,9 +319,7 @@ export function VideoCard({file}) {
<CardContent {...s}>
<CardHeader>
<Container textAlign='left'>
<Link to={video_url} className="no-link-underscore card-link">
<p {...s}>{cardTitleWrapper(file.title || file.name || video.stem || video.video_path)}</p>
</Link>
{header}
</Container>
</CardHeader>
<CardDescription>

5
app/src/hooks/customHooks.js
View file

@ -735,6 +735,11 @@ export const StatusProvider = (props) => {
</StatusContext.Provider>
}
export const useStatusFlag = (flag) => {
const {status} = useContext(StatusContext);
return status && status['flags'] && status['flags'].indexOf(flag) >= 0;
}
export const useVideoStatistics = () => {
const [statistics, setStatistics] = useState({});

30
modules/archive/__init__.py
View file

@ -11,11 +11,11 @@ from wrolpi.common import logger, register_modeler, register_refresh_cleanup, li
from wrolpi.db import optional_session, get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from .errors import InvalidArchive
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
from . import lib
from .api import bp # noqa
from .errors import InvalidArchive
from .lib import is_singlefile_file, get_title_from_html
from .models import Archive, Domain
@ -193,34 +193,6 @@ async def archive_modeler():
@register_refresh_cleanup
@limit_concurrent(1)
def archive_cleanup():
# Process all Archives that have not been validated. Delete any that no longer exist, or are not real Archives.
# Validate all other Archives.
logger.info('Searching for invalid Archives')
offset = 0
limit = 100
while True:
with get_db_session(commit=True) as session:
archives: List[Archive] = session.query(Archive).order_by(Archive.id).limit(limit).offset(offset)
processed = 0
deleted = 0
for archive in archives:
processed += 1
singlefile_path = archive.singlefile_path
if not singlefile_path:
archive.file_group.model = None
session.delete(archive)
deleted += 1
continue
if not lib.is_singlefile_file(singlefile_path):
# Archive is not a real archive.
archive.singlefile_file.model = None
session.delete(archive)
deleted += 1
if processed < limit:
# Validated all archives.
break
offset += (limit - deleted)
with get_db_session(commit=True) as session:
# Remove any Domains without any Archives.
domain_ids = [i[0] for i in session.execute('SELECT DISTINCT domain_id FROM archive') if i[0]]

5
modules/archive/api.py
View file

@ -4,6 +4,7 @@ from http import HTTPStatus
from sanic import response, Request
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from sanic_ext.extensions.openapi.definitions import Response
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
from wrolpi.errors import ValidationError
@ -43,7 +44,9 @@ async def delete_archive(_: Request, archive_ids: str):
@bp.get('/domains')
async def fetch_domains(_: Request):
@openapi.summary('Get a list of all Domains and their Archive statistics')
@openapi.response(200, schema.GetDomainsResponse, "The list of domains")
async def get_domains(_: Request):
domains = lib.get_domains()
return json_response({'domains': domains, 'totals': {'domains': len(domains)}})

8
modules/archive/conftest.py
View file

@ -1,6 +1,7 @@
import datetime
import json
import pathlib
from typing import List
from uuid import uuid4
import pytest
@ -28,10 +29,12 @@ def archive_factory(test_session, archive_directory, make_files_structure):
now = time_generator()
def _(domain: str = None, url: str = None, title: str = 'NA', contents: str = None,
singlefile_contents: str = None) -> Archive:
singlefile_contents: str = None, tag_names: List[str] = None) -> Archive:
if domain:
assert '/' not in domain
tag_names = tag_names or list()
title = title or str(uuid4())
domain_dir = domain or title
domain_dir = archive_directory / domain_dir
@ -77,6 +80,9 @@ def archive_factory(test_session, archive_directory, make_files_structure):
archive.domain = domain
archive.validate()
for tag_name in tag_names:
archive.add_tag(tag_name)
return archive
return _

10
modules/archive/lib.py
View file

@ -20,7 +20,6 @@ from wrolpi.common import get_media_directory, logger, extract_domain, chdir, es
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy
from wrolpi.files.lib import handle_file_group_search_results
from wrolpi.tags import tag_names_to_file_group_sub_select
from wrolpi.vars import DOCKERIZED, PYTEST
@ -356,14 +355,15 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
def get_domains():
with get_db_curs() as curs:
stmt = '''
SELECT domains.domain AS domain, COUNT(a.id) AS url_count
SELECT domains.domain AS domain, COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size
FROM domains
LEFT JOIN archive a on domains.id = a.domain_id
LEFT JOIN archive a on domains.id = a.domain_id
LEFT JOIN file_group fg on fg.id = a.file_group_id
GROUP BY domains.domain
ORDER BY domains.domain
'''
curs.execute(stmt)
domains = list(map(dict, curs.fetchall()))
domains = [dict(i) for i in curs.fetchall()]
return domains
@ -421,6 +421,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
tags_stmt, params_ = tag_names_to_file_group_sub_select(tag_names)
params.update(params_)
wheres.append(f'fg.id = ANY({tags_stmt})')
joins = ['LEFT JOIN file_group fg ON fg.id = a.file_group_id']
if search_str and headline:
headline = ''',
@ -455,6 +456,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
'''.strip()
logger.debug(stmt, params)
from wrolpi.files.lib import handle_file_group_search_results
results, total = handle_file_group_search_results(stmt, params)
return results, total

2
modules/archive/models.py
View file

@ -11,11 +11,11 @@ from sqlalchemy.orm.collections import InstrumentedList
from wrolpi.common import ModelHelper, Base, logger
from wrolpi.dates import TZDateTime
from modules.archive import InvalidArchive
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
from wrolpi.tags import Tag, TagFile
from wrolpi.vars import PYTEST
from .errors import InvalidArchive
logger = logger.getChild(__name__)

8
modules/archive/schema.py
View file

@ -19,8 +19,14 @@ class ArchiveDict:
@dataclass
class DomainDict:
id: int
domain: str
url_count: int
size: int
@dataclass
class GetDomainsResponse:
domains: List[DomainDict]
@dataclass

23
modules/archive/test/test_api.py
View file

@ -1,6 +1,8 @@
import json
from http import HTTPStatus
import pytest
from modules.archive import lib
@ -79,6 +81,26 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory):
"""Tagged Archives can be searched."""
tag = tag_factory()
archive_factory(domain='example.com', tag_names=[tag.name, ])
test_session.commit()
content = {'search_str': '', 'domain': 'example.com', 'tag_names': [tag.name, ]}
request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups']
assert response.json['totals']['file_groups'] == 1
content = {'search_str': '', 'domain': 'example.com', 'tag_names': ['does not exist', ]}
request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == []
assert response.json['totals']['file_groups'] == 0
def test_archives_search_headline(test_session, archive_directory, archive_factory, test_client):
"""Headlines can be requested."""
archive_factory('example.com', 'https://example.com/one', 'my archive', 'foo bar qux')
@ -179,6 +201,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'][0]['domain'] == 'example.com'
assert response.json['domains'][0]['size'] == 116
# Deleting works.
request, response = test_client.delete(f'/api/archive/{archive1.id}')

48
modules/archive/test/test_lib.py
View file

@ -12,7 +12,7 @@ from modules.archive import lib
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, do_archive, get_domains
from modules.archive.models import Archive, Domain
from wrolpi.db import get_db_session
from wrolpi.files.lib import refresh_files
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.root_api import CustomJSONEncoder
from wrolpi.test.common import skip_circleci
@ -94,7 +94,7 @@ async def test_archive_title(test_session, archive_factory, singlefile_contents_
test_session.query(FileGroup).delete()
test_session.commit()
await refresh_files()
await files_lib.refresh_files()
assert test_session.query(FileGroup).count() == 1
assert test_session.query(Archive).count() == 1
@ -108,7 +108,7 @@ async def test_archive_title(test_session, archive_factory, singlefile_contents_
# Delete JSON file.
archive1.readability_json_path.unlink()
test_session.delete(archive1)
await refresh_files()
await files_lib.refresh_files()
archive1: Archive = test_session.query(Archive).one()
assert archive1.url == 'https://example.com/singlefile-url'
@ -117,7 +117,7 @@ async def test_archive_title(test_session, archive_factory, singlefile_contents_
# Clear out title and URL from Singlefile.
archive1.singlefile_path.write_text(singlefile_contents_factory('', ''))
test_session.delete(archive1)
await refresh_files()
await files_lib.refresh_files()
# No title/URL could be found.
archive1: Archive = test_session.query(Archive).one()
@ -207,7 +207,7 @@ async def test_fills_contents_with_refresh(test_client, test_session, archive_fa
assert not archive4.file_group.d_text
# Fill the contents.
await refresh_files()
await files_lib.refresh_files()
# The archives will be renamed with their title.
archive1, archive2, archive3, archive4 = test_session.query(Archive).order_by(Archive.id)
assert not archive4.file_group.d_text
@ -536,7 +536,7 @@ async def test_refresh_archives_index(test_session, test_directory, test_client,
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
})
await refresh_files()
await files_lib.refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
@ -545,32 +545,22 @@ async def test_refresh_archives_index(test_session, test_directory, test_client,
assert archive.file_group.indexed is True
def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure):
"""
Invalid Archives should be removed during refresh.
"""
*_, bogus_path = make_files_structure({
'2022-09-04-16-20-11_The Title.html': '<html></html>',
@pytest.mark.asyncio
async def test_refresh_archives_deleted_singlefile(test_session, make_files_structure, singlefile_contents_factory):
"""Removing a Singlefile file from a FileGroup makes that group no longer an Archive."""
singlefile, readability = make_files_structure({
'2022-09-04-16-20-11_The Title.html': singlefile_contents_factory(),
'2022-09-04-16-20-11_The Title.readability.json': '{"url": "https://example.com"}',
'bogus file': None,
})
test_client.post('/api/files/refresh')
assert test_session.query(FileGroup).count() == 2 # One bogus file, one archive group.
assert test_session.query(Archive).count() == 1 # Only one real archive.
await files_lib.refresh_files()
assert test_session.query(FileGroup).one().model == 'archive'
assert test_session.query(Archive).count() == 1
# Create an invalid Archive, it should be removed during refresh.
bogus_file: FileGroup = test_session.query(FileGroup).filter_by(primary_path=bogus_path).one()
bogus_file.model = 'archive'
archive = Archive(file_group=bogus_file)
test_session.add(archive)
test_session.commit()
assert test_session.query(FileGroup).count() == 2
assert test_session.query(Archive).count() == 2
test_client.post('/api/files/refresh')
assert test_session.query(FileGroup).count() == 2, 'More files in DB than we created'
assert test_session.query(Archive).count() == 1, 'The bogus Archive was not removed'
assert bogus_file.model is None, 'Model was not removed'
# Remove singlefile, FileGroup is no longer an Archive.
singlefile.unlink()
await files_lib.refresh_files()
assert test_session.query(FileGroup).one().model is None
assert test_session.query(Archive).count() == 0
@pytest.mark.parametrize(

3
modules/videos/__init__.py
View file

@ -37,6 +37,9 @@ async def video_modeler():
session.add(video)
session.flush([video])
video_id = video.id
# Extract ffprobe data.
await video.get_ffprobe_json()
# Validate and index subtitles.
video.validate()
except Exception as e:
if PYTEST:

9
modules/videos/channel/test/test_api.py
View file

@ -1,3 +1,4 @@
import asyncio
import json
import pathlib
import tempfile
@ -275,16 +276,14 @@ async def test_channel_download_requires_refresh(test_session, download_channel,
vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg')
vid.url = 'https://example.com/1'
d = download_channel.get_download()
d.next_download = None
test_session.commit()
assert not d.next_download
def assert_refreshed(expected: bool):
channel: Channel = test_session.query(Channel).one()
assert channel.refreshed == expected
if expected:
assert channel.info_json
else:
assert not channel.info_json
assert bool(channel.info_json) == expected
assert_refreshed(False)
test_session.commit()
@ -307,6 +306,8 @@ async def test_channel_download_requires_refresh(test_session, download_channel,
'channel_id': 'the id', 'id': 'the id'}
await video_download_manager.do_downloads()
await video_download_manager.wait_for_all_downloads()
# Give time for background tasks to finish. :(
await asyncio.sleep(1)
# Channel was refreshed before downloading videos.
assert_refreshed(True)

48
modules/videos/common.py
View file

@ -1,13 +1,15 @@
import asyncio
import json
import os
import pathlib
import re
import shlex
import subprocess
import tempfile
from datetime import timedelta
from decimal import Decimal
from pathlib import Path
from typing import Optional, Tuple
from typing import Optional, Tuple, Union
import PIL
from PIL import Image
@ -165,6 +167,50 @@ def is_valid_poster(poster_path: Path) -> bool:
return False
async def ffprobe_json(video_path: Union[Path, str]) -> dict:
"""Extract video file metadata using ffprobe.
>>> ffprobe_json('video')
{
'chapters': [],
'format': {},
'streams': [
{},
],
}
"""
cmd = f'{FFPROBE_BIN}' \
f' -print_format json' \
f' -loglevel quiet' \
f' -show_streams' \
f' -show_format' \
f' -show_chapters' \
f' {shlex.quote(str(video_path.absolute()))}'
try:
proc = await asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = await proc.communicate()
except Exception as e:
logger.error(f'Failed to run ffprobe json', exc_info=e)
raise
if proc.returncode != 0:
raise RuntimeError(f'Got non-zero exit code: {proc.returncode}')
try:
content = json.loads(stdout.decode().strip())
except Exception as e:
logger.debug(stdout.decode())
logger.error(f'Failed to load ffprobe json', exc_info=e)
raise
if content == dict():
# Data is empty, video may be corrupt.
raise RuntimeError('ffprobe data was empty')
return content
def get_video_duration(video_path: Path) -> int:
"""Get the duration of a video in seconds. Do this using ffprobe."""
if not isinstance(video_path, Path):

16
modules/videos/downloader.py
View file

@ -19,11 +19,11 @@ from wrolpi.db import get_db_session
from wrolpi.db import optional_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from .errors import UnknownChannel, ChannelURLEmpty
from wrolpi.files.lib import glob_shared_stem
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import get_no_channel_directory, get_videos_directory, update_view_counts
from .errors import UnknownChannel, ChannelURLEmpty
from .lib import get_downloader_config
from .models import Video, Channel
from .schema import ChannelPostRequest
@ -311,6 +311,20 @@ class VideoDownloader(Downloader, ABC):
if name not in existing_names:
video.add_tag(name)
# Check that video is valid.
await video.get_ffprobe_json()
if not video.get_streams_by_codec_type('video'):
return DownloadResult(
success=False,
error='Video was downloaded but did not contain video stream',
)
if not video.get_streams_by_codec_type('audio'):
return DownloadResult(
success=False,
error='Video was downloaded but did not contain audio stream',
)
session.commit()
except UnrecoverableDownloadError:
raise
except yt_dlp.utils.UnsupportedError as e:

6
modules/videos/lib.py
View file

@ -119,7 +119,11 @@ def validate_video(video: Video, channel_generate_poster: bool):
if video_path and not video.duration:
# Duration was not retrieved during poster generation.
video.duration = get_video_duration(video_path)
if video.ffprobe_json and (video_streams := video.get_streams_by_codec_type('video')):
video.duration = float(video_streams[0]['duration'])
else:
# Slowest method.
video.duration = get_video_duration(video_path)
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:
# Caption file was deleted, clear out old captions.

55
modules/videos/models.py
View file

@ -8,12 +8,12 @@ from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, Fore
from sqlalchemy.orm import relationship, Session, deferred
from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.errors import UnknownVideo
from wrolpi.captions import read_captions
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task
from wrolpi.dates import now, TZDateTime
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download, download_manager
from modules.videos.errors import UnknownVideo
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
@ -35,6 +35,7 @@ class Video(ModelHelper, Base):
url = Column(String)
view_count = Column(Integer)
viewed = Column(TZDateTime)
ffprobe_json = deferred(Column(JSON))
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
@ -44,7 +45,7 @@ class Video(ModelHelper, Base):
def __repr__(self):
v = None
if self.video_path:
v = self.video_path.relative_to(get_media_directory())
v = repr(str(self.video_path.relative_to(get_media_directory())))
return f'<Video id={self.id} title={repr(self.file_group.title)} path={v} channel={self.channel_id} ' \
f'source_id={repr(self.source_id)}>'
@ -55,12 +56,20 @@ class Video(ModelHelper, Base):
if self.channel:
channel = dict(id=self.channel.id, name=self.channel.name)
codec_names = []
codec_types = []
if self.ffprobe_json:
codec_names = [i['codec_name'] for i in self.ffprobe_json['streams']]
codec_types = [i['codec_type'] for i in self.ffprobe_json['streams']]
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
d['video'] = dict(
caption=self.file_group.d_text,
caption_files=self.caption_files,
channel=channel,
channel_id=self.channel_id,
codec_names=codec_names,
codec_types=codec_types,
description=self.file_group.c_text or self.get_video_description(),
duration=self.duration,
id=self.id,
@ -198,6 +207,8 @@ class Video(ModelHelper, Base):
validate_video(self, self.channel.generate_posters if self.channel else False)
except Exception as e:
logger.warning(f'Failed to validate video {self}', exc_info=e)
if PYTEST:
raise
self.file_group.model = Video.__tablename__
self.file_group.a_text = self.file_group.title
@ -289,6 +300,46 @@ class Video(ModelHelper, Base):
tag = Tag.find_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name
return self.file_group.add_tag(tag)
async def get_ffprobe_json(self) -> dict:
"""Return the ffprobe json object if previously stored.
Runs ffprobe if this data does not yet exist."""
if not self.video_path:
raise RuntimeError(f'Cannot get ffprobe json without video file: {self}')
if not self.ffprobe_json:
from modules.videos.common import ffprobe_json
self.ffprobe_json = await ffprobe_json(self.video_path)
self.flush()
return self.ffprobe_json
def get_streams_by_codec_name(self, codec_name: str) -> List[dict]:
"""Return all data about all streams which match the codec_name.
>>> video = Video()
>>> video.get_streams_by_codec_name('h264')
[ {'codec_name': 'h264', ...} ]
"""
if not self.ffprobe_json:
raise RuntimeError(f'ffprobe data has not been extracted, call Video.get_ffprobe_json().')
streams = [i for i in self.ffprobe_json['streams'] if i['codec_name'] == codec_name]
return streams
def get_streams_by_codec_type(self, codec_type: str) -> List[dict]:
"""Return all data about all streams which match the codec_type.
>>> video = Video()
>>> video.get_streams_by_codec_type('video')
[ {'codec_type': 'video', ...} ]
"""
if not self.ffprobe_json:
raise RuntimeError(f'ffprobe data has not been extracted, call Video.get_ffprobe_json().')
streams = [i for i in self.ffprobe_json['streams'] if i['codec_type'] == codec_type]
return streams
class Channel(ModelHelper, Base):
__tablename__ = 'channel'

26
modules/videos/test/test_common.py
View file

@ -9,6 +9,7 @@ from PIL import Image
from modules.videos.models import Channel, Video
from wrolpi.common import get_absolute_media_path
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.files import lib as files_lib
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import convert_image, update_view_counts, get_video_duration, generate_video_poster, is_valid_poster
@ -204,3 +205,28 @@ def test_check_for_video_corruption(video_file, test_directory):
assert common.check_for_video_corruption(video_file) is True
mock_subprocess.run().stderr = b'Some stderr is fine'
assert common.check_for_video_corruption(video_file) is False
@pytest.mark.asyncio
async def test_ffprobe_json(video_file, corrupted_video_file):
content = await common.ffprobe_json(video_file)
assert not content['chapters']
assert content['format']['duration'] == '5.312000'
assert content['format']['size'] == '1056318'
assert content['streams']
assert content['streams'][0]['codec_name'] == 'h264'
with pytest.raises(RuntimeError):
await common.ffprobe_json(corrupted_video_file)
@pytest.mark.asyncio
async def test_video_ffprobe_json(test_session, video_file):
"""ffprobe data is extracted when a video is modeled."""
with mock.patch('modules.videos.lib.get_video_duration') as mock_get_video_duration:
mock_get_video_duration.side_effect = Exception('duration should be from ffprobe json')
await files_lib.refresh_files()
video = test_session.query(Video).one()
assert video.ffprobe_json
assert video.duration

2
modules/videos/test/test_downloader.py
View file

@ -366,6 +366,8 @@ async def test_download_result(test_session, test_directory, video_download_mana
mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
video_download_manager.create_download('https://example.com', video_downloader.name)
await video_download_manager.wait_for_all_downloads()
# Sleep to allow background tasks to finish.
await asyncio.sleep(1)
download: Download = test_session.query(Download).one()
assert download.url == 'https://example.com'

15
modules/videos/test/test_models.py Normal file
View file

@ -0,0 +1,15 @@
import pytest
@pytest.mark.asyncio()
async def test_ffprobe_stream_methods(simple_video):
"""ffprobe data can be used to get data about specific streams in the video file."""
assert await simple_video.get_ffprobe_json()
assert simple_video.get_streams_by_codec_type('video')
assert simple_video.get_streams_by_codec_type('audio')
assert simple_video.get_streams_by_codec_type('subtitle')
assert simple_video.get_streams_by_codec_name('h264')
assert simple_video.get_streams_by_codec_name('aac')
assert simple_video.get_streams_by_codec_name('mov_text')

1
pi-gen/stage2/04-wrolpi/01-packages
View file

@ -50,6 +50,7 @@ libtool
libxml2-dev
lua5.1
mapnik-utils
netcat
network-manager
nginx-doc
nginx-full

132
scripts/find_corrupt_videos.py Executable file
View file

@ -0,0 +1,132 @@
#! /usr/bin/env python3
"""
Searches the DB for Video records which do not have both video and audio streams. If a Video still exists at its URL,
it will be automatically re-downloaded. If a Video cannot be re-downloaded (it has been deleted from its host) you
will be prompted to delete it.
"""
import argparse
import asyncio
import logging
import sys
from typing import List
from yt_dlp.utils import YoutubeDLError
from modules.videos.downloader import VideoDownloader, extract_info
from modules.videos.models import Video
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import download_manager
logger = logging.getLogger()
def get_total_corrupted_videos() -> int:
with get_db_curs() as curs:
stmt = '''
WITH all_videos AS (
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
from video v
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
group by v.id)
SELECT COUNT(all_videos.id)
FROM all_videos
WHERE NOT all_videos.codec_types @> '{audio,video}'::TEXT[]
'''
curs.execute(stmt)
return int(curs.fetchone()[0])
def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
params = dict(limit=limit, offset=offset)
with get_db_curs() as curs:
stmt = '''
WITH all_videos AS (
-- Extract 'codec_type' from all 'stream' objects of each video.
-- {'streams': [{'codec_type': 'video'}, {'codec_type': 'audio'}]} -> {video,audio}
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
from video v
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
group by v.id)
SELECT all_videos.id AS video_id, fg.primary_path, all_videos.codec_types
FROM all_videos
LEFT JOIN video v ON v.id = all_videos.id
LEFT JOIN public.file_group fg on fg.id = v.file_group_id
-- Filter out videos that have video AND audio streams.
WHERE NOT all_videos.codec_types @> '{video,audio}'::TEXT[]
AND fg.primary_path like '/media/wrolpi/videos/%%'
ORDER BY all_videos.id
LIMIT %(limit)s
OFFSET %(offset)s
'''
curs.execute(stmt, params)
return [dict(i) for i in curs.fetchall()]
def iterate_corrupt_videos() -> List[dict]:
limit = 20
offset = 0
while True:
videos = get_corrupt_videos(limit, offset)
offset += limit
if len(videos) < limit:
# Ran out of corrupt videos.
break
yield videos
def confirm(msg: str) -> bool:
while True:
input_ = input(msg)
if input_ == 'y':
return True
if input_ == 'q':
print('Quitting...')
sys.exit(0)
if not input_ or input_ == 'n':
return False
async def find_corrupt_videos():
total_corrupted = get_total_corrupted_videos()
count = 0
with get_db_session() as session:
for chunk in iterate_corrupt_videos():
video_ids = [i['video_id'] for i in chunk]
videos = session.query(Video).filter(Video.id.in_(video_ids)).all()
for video in videos:
print()
print(f'Video (remaining: {total_corrupted - count}): {video}')
count += 1
streams = (await video.get_ffprobe_json())['streams']
codec_names = [i['codec_name'] for i in streams]
print(f'codecs: {codec_names}')
url = video.url
if url:
try:
extract_info(url)
print(f'Re-downloading')
download_manager.create_download(url, VideoDownloader.name, session=session)
video.delete()
session.commit()
continue
except YoutubeDLError:
logger.error(f'Video cannot be downloaded: {url}')
if confirm(f'Delete? (y/N)'):
video.delete()
session.commit()
continue
if confirm('Delete? (y/N)'):
video.delete()
session.commit()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
args = parser.parse_args()
loop = asyncio.get_event_loop()
coro = find_corrupt_videos()
loop.run_until_complete(coro)

1
scripts/install_debian_11.sh
View file

@ -23,6 +23,7 @@ apt-get install -y \
kiwix \
kiwix-tools \
libpq-dev \
netcat \
network-manager \
nginx-doc \
nginx-full \

1
scripts/install_raspberrypios.sh
View file

@ -23,6 +23,7 @@ apt-get install -y \
kiwix \
kiwix-tools \
libpq-dev \
netcat \
network-manager \
nginx-doc \
nginx-full \

13
scripts/start_kiwix_serve.sh
View file

@ -1,16 +1,25 @@
#! /usr/bin/env bash
# This file imports all *.zim files in the media directory, then starts kiwix-serve.
# This file imports all *.zim files in the Zim media directory, then starts kiwix-serve.
[ ! -d /media/wrolpi ] && echo "Cannot start kiwix without media directory mounted" && exit 1
LIBRARY=/media/wrolpi/zims/library.xml
# Wait for the zims directory to be mounted.
COUNT=0
while [ ${COUNT} -lt 30 ]; do
if [ -d /media/wrolpi/zims ]; then
break
fi
sleep 1
done
# Create the zim directory, if necessary.
[ ! -d /media/wrolpi/zims ] && mkdir /media/wrolpi/zims
# We will replace the library file.
rm -f ${LIBRARY}
# Find all Zim files and import them.
find /media/wrolpi -iname '*.zim' -exec kiwix-manage ${LIBRARY} add {} \;
find /media/wrolpi/zims -iname '*.zim' -exec kiwix-manage ${LIBRARY} add {} \;
[ ! -f ${LIBRARY} ] && echo "Could not find any Zim files to import" && exit 2

12
wrolpi/common.py
View file

@ -34,6 +34,7 @@ from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from wrolpi import dates
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
@ -114,6 +115,8 @@ __all__ = [
'limit_concurrent',
'logger',
'make_media_directory',
'split_lines_by_length',
'slow_logger',
'match_paths_to_suffixes',
'minimize_dict',
'native_only',
@ -163,6 +166,10 @@ class ModelHelper:
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
raise NotImplementedError('This model has not defined this method.')
def flush(self):
"""A convenience function which flushes this record using its DB Session."""
Session.object_session(self).flush([self])
def get_model_by_table_name(table_name):
"""Find a model by its table name."""
@ -1079,6 +1086,7 @@ def register_refresh_cleanup(func: callable):
async def apply_refresh_cleanup():
for func in REFRESH_CLEANUP:
start = dates.now()
try:
logger_.info(f'Applying refresh cleanup {func.__name__}')
func()
@ -1086,6 +1094,9 @@ async def apply_refresh_cleanup():
logger.error(f'Refresh cleanup {func.__name__} failed!', exc_info=e)
if PYTEST:
raise
finally:
elapsed = (dates.now() - start).total_seconds()
logger.warning(f'After refresh cleanup {func.__name__} took {int(elapsed)} seconds')
# Sleep to catch cancel.
await asyncio.sleep(0)
@ -1167,6 +1178,7 @@ def timer(name):
@contextlib.contextmanager
def slow_logger(max_seconds: int, message: str, logger__=logger, level=logging.WARNING):
"""Only logs when the context duration exceeds the `max_seconds`."""
before = now()
try:
yield

13
wrolpi/conftest.py
View file

@ -13,7 +13,7 @@ import zipfile
from abc import ABC
from datetime import datetime
from itertools import zip_longest
from typing import List, Callable, Union, Dict, Sequence
from typing import List, Callable, Dict, Sequence
from typing import Tuple, Set
from unittest import mock
from unittest.mock import MagicMock
@ -241,12 +241,21 @@ def test_downloader(test_download_manager):
@pytest.fixture
def video_file(test_directory) -> pathlib.Path:
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
destination = test_directory / f'{uuid4()}.mp4'
destination = test_directory / f'video-{uuid4()}.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
yield destination
@pytest.fixture
def corrupted_video_file(test_directory) -> pathlib.Path:
"""Return a copy of a corrupted video file in the `test_directory`."""
destination = test_directory / f'corrupted-{uuid4()}.mp4'
shutil.copy(PROJECT_DIR / 'test/corrupted.mp4', destination)
yield destination
@pytest.fixture
def video_file_factory(test_directory):
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""

4
wrolpi/events.py
View file

@ -22,8 +22,8 @@ class Events:
send_event('global_refresh_started', message, subject='refresh')
@staticmethod
def send_global_refresh_completed(message: str = None):
send_event('global_refresh_completed', message, subject='refresh')
def send_refresh_completed(message: str = None):
send_event('refresh_completed', message, subject='refresh')
@staticmethod
def send_global_refresh_modeling_completed(message: str = None):

2
wrolpi/files/api.py
View file

@ -389,7 +389,7 @@ async def post_upload(request: Request):
if chunk_num == total_chunks:
# File upload is complete.
del UPLOADED_FILES[output_str]
background_task(lib.refresh_files([output]))
background_task(lib.refresh_files([output.parent]))
return response.empty(HTTPStatus.CREATED)
# Request the next chunk.

10
wrolpi/files/lib.py
View file

@ -46,7 +46,7 @@ logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', 'estimate_search',
'move', 'rename', 'delete_directory']
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
@optional_session
@ -407,6 +407,8 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
need_index = [i[0] for i in curs.fetchall() if i[1] is False]
if need_index:
refresh_logger.info(f'Invalidated indexes of {len(need_index)} file groups')
else:
logger.debug(f'Upserted {len(chunk)} files')
if non_primary_files:
# New files may have been added which change what primary paths exist. Delete any file_groups which
@ -561,8 +563,8 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
if refreshing_all_files:
# Only set refresh_complete flag if all files have been refreshed.
flags.refresh_complete.set()
if send_events:
Events.send_global_refresh_completed()
if send_events:
Events.send_refresh_completed()
async def apply_indexers():
@ -888,7 +890,7 @@ def group_files_by_stem(files: List[pathlib.Path], pre_sorted: bool = False) ->
@param files: A list of paths to be grouped. All paths must be in the same directory!
@param pre_sorted: This function requires the files to be sorted, it will sort them by default.
"""
files = sorted(files) if not pre_sorted else files
files = sorted(files) if not pre_sorted else files.copy()
file = files.pop(0)
group = [file, ]
prev_stem, _ = split_path_stem_and_suffix(file)

19
wrolpi/files/test/test_lib.py
View file

@ -184,10 +184,10 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
{'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
]},
{'primary_path': bar, 'idempotency': idempotency,
{'primary_path': bar, 'idempotency': idempotency, 'indexed': False,
'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}]
},
{'primary_path': baz, 'idempotency': idempotency,
{'primary_path': baz, 'idempotency': idempotency, 'indexed': False,
'files': [{'path': baz, 'size': 8, 'suffix': '.txt', 'mimetype': 'text/plain'}]
},
])
@ -211,10 +211,10 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
{'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
]},
{'primary_path': bar, 'idempotency': idempotency,
{'primary_path': bar, 'idempotency': idempotency, 'indexed': False,
'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}],
},
{'primary_path': baz, 'idempotency': idempotency,
{'primary_path': baz, 'idempotency': idempotency, 'indexed': False,
'files': [{'path': baz, 'size': 7, 'suffix': '.txt', 'mimetype': 'text/plain'}],
},
])
@ -224,6 +224,17 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
lib._upsert_files([video_file, bar, baz], idempotency)
video_file_group: FileGroup = test_session.query(FileGroup).filter_by(primary_path=str(video_file)).one()
assert len(video_file_group.files) == 1, 'SRT file was not removed from files'
assert_file_groups([
# Video is no longer indexed because SRT was removed.
{'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': False,
'files': [{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'}]},
{'primary_path': bar, 'idempotency': idempotency, 'indexed': False,
'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}],
},
{'primary_path': baz, 'idempotency': idempotency, 'indexed': False,
'files': [{'path': baz, 'size': 7, 'suffix': '.txt', 'mimetype': 'text/plain'}],
},
])
@pytest.mark.asyncio

2
wrolpi/test/test_events.py
View file

@ -28,7 +28,7 @@ def test_events_api_feed(test_session, test_client, example_pdf):
assert (result := response.json.get('events'))
expected = [
dict(event='global_refresh_completed', subject='refresh'),
dict(event='refresh_completed', subject='refresh'),
dict(event='global_after_refresh_completed', subject='refresh'),
dict(event='global_refresh_indexing_completed', subject='refresh'),
dict(event='global_refresh_modeling_completed', subject='refresh'),