rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 02:48:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Creating new /api/archive/upload which accepts a file from the SingleFile browser extension and converts it to an Archive.

Browse source
This commit is contained in:
Roland 2025-01-26 12:44:16 -07:00
parent 01338e4d89
commit 531ec3d65e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
62 changed files with 1000 additions and 562 deletions
Download patch file Download diff file Expand all files Collapse all files

134
app/src/DashboardPage.js
View file

@ -1,4 +1,11 @@
import {ErrorMessage, LoadStatistic, PageContainer, SearchResultsInput, useTitle} from "./components/Common";
import {
ErrorMessage,
LoadStatistic,
PageContainer,
RefreshHeader,
SearchResultsInput,
useTitle
} from "./components/Common";
import React, {useContext, useState} from "react";
import {Media, SettingsContext, StatusContext} from "./contexts/contexts";
import {DownloadMenu} from "./components/Download";
@ -25,8 +32,8 @@ import {TagsDashboard} from "./Tags";
import {Upload} from "./components/Upload";
import {SearchView, useSearch, useSearchSuggestions} from "./components/Search";
import {KiwixRestartMessage, OutdatedZimsMessage} from "./components/Zim";
import {useSearchFilter, useWROLMode} from "./hooks/customHooks";
import {FileSearchFilterButton} from "./components/Files";
import {useSearchFilter, useSearchRecentFiles, useWROLMode} from "./hooks/customHooks";
import {FileCards, FileSearchFilterButton} from "./components/Files";
import {DateSelectorButton} from "./components/DatesSelector";
import {useCalculators} from "./components/Calculators";
import {evaluate} from "mathjs";
@ -234,6 +241,72 @@ export function Getters() {
</>
}
function DashboardStatus() {
const {status} = useContext(StatusContext);
let percent = 0;
let load = {};
let cores = 0;
let pending_downloads = '?';
if (status && status['cpu_stats']) {
percent = status['cpu_stats']['percent'];
load = status['load_stats'];
cores = status['cpu_stats']['cores'];
}
const {downloads} = status;
if (!_.isEmpty(downloads)) {
pending_downloads = downloads && downloads['disabled'] ? 'x' : downloads['pending'];
}
let bandwidths = <ProgressPlaceholder/>;
if (status && status['nic_bandwidth_stats']) {
bandwidths = Object.entries(status['nic_bandwidth_stats'])
.map(([name, bandwidth]) => <BandwidthProgressCombined key={name} bandwidth={bandwidth}/>);
} else if (window.apiDown) { // apiDown is set in useStatus
bandwidths = null;
}
return <Segment>
<Link to='/admin/status'>
<Header as='h2'>Status</Header>
<CPUUsageProgress value={percent} label='CPU Usage'/>
<StatisticGroup size='mini'>
<LoadStatistic label='1 Min. Load' value={load['minute_1']} cores={cores}/>
<LoadStatistic label='5 Min. Load' value={load['minute_5']} cores={cores}/>
<LoadStatistic label='15 Min. Load' value={load['minute_15']} cores={cores}/>
</StatisticGroup>
<Header as='h3'>Bandwidth</Header>
{bandwidths}
</Link>
<Divider style={{marginTop: '3em'}}/>
<Link to='/admin'>
<StatisticGroup size='mini'>
<Statistic label='Downloading' value={pending_downloads}/>
</StatisticGroup>
</Link>
</Segment>;
}
function DashboardRecentFiles() {
const {searchFiles, loading, fetchFiles} = useSearchRecentFiles();
return <Segment>
<RefreshHeader
header='Recently Viewed Files'
popupContents='Fetch the most recent files again'
onRefresh={fetchFiles}
/>
<FileCards files={searchFiles} loading={loading}/>
</Segment>
}
export function DashboardPage() {
const navigate = useNavigate();
@ -284,6 +357,7 @@ export function DashboardPage() {
<TagsDashboard/>
<DashboardStatus/>
<DashboardCalculators/>
<DashboardRecentFiles/>
</React.Fragment>;
if (searchStr || (activeTags && activeTags.length > 0)) {
// User has submitted and wants full search.
@ -370,56 +444,4 @@ export function DashboardPage() {
{!searchStr && <FlagsMessages/>}
{body}
</PageContainer>
}
function DashboardStatus() {
const {status} = useContext(StatusContext);
let percent = 0;
let load = {};
let cores = 0;
let pending_downloads = '?';
if (status && status['cpu_stats']) {
percent = status['cpu_stats']['percent'];
load = status['load_stats'];
cores = status['cpu_stats']['cores'];
}
const {downloads} = status;
if (!_.isEmpty(downloads)) {
pending_downloads = downloads && downloads['disabled'] ? 'x' : downloads['pending'];
}
let bandwidths = <ProgressPlaceholder/>;
if (status && status['nic_bandwidth_stats']) {
bandwidths = Object.entries(status['nic_bandwidth_stats'])
.map(([name, bandwidth]) => <BandwidthProgressCombined key={name} bandwidth={bandwidth}/>);
} else if (window.apiDown) { // apiDown is set in useStatus
bandwidths = null;
}
return <Segment>
<Link to='/admin/status'>
<Header as='h2'>Status</Header>
<CPUUsageProgress value={percent} label='CPU Usage'/>
<StatisticGroup size='mini'>
<LoadStatistic label='1 Min. Load' value={load['minute_1']} cores={cores}/>
<LoadStatistic label='5 Min. Load' value={load['minute_5']} cores={cores}/>
<LoadStatistic label='15 Min. Load' value={load['minute_15']} cores={cores}/>
</StatisticGroup>
<Header as='h3'>Bandwidth</Header>
{bandwidths}
</Link>
<Divider style={{marginTop: '3em'}}/>
<Link to='/admin'>
<StatisticGroup size='mini'>
<Statistic label='Downloading' value={pending_downloads}/>
</StatisticGroup>
</Link>
</Segment>;
}
}

9
app/src/Events.js
View file

@ -92,6 +92,15 @@ function handleEvents(events) {
eventToast('Config Save Failed', message, 'error');
}
if (event === 'upload_archive') {
eventToast(
'Archive Uploaded',
message,
'success',
5000,
() => window.open(url, '_self'));
}
if (subject) {
newestEvents[subject] = dt;
}

35
app/src/api.js
View file

@ -222,6 +222,22 @@ export async function getVideo(video_id) {
return [data['file_group'], data['prev'], data['next']];
}
export async function getVideoComments(videoId) {
const response = await apiGet(`${VIDEOS_API}/video/${videoId}/comments`);
let data = await response.json();
return {
comments: data.comments,
}
}
export async function getVideoCaptions(videoId) {
const response = await apiGet(`${VIDEOS_API}/video/${videoId}/captions`);
let data = await response.json();
return {
captions: data.captions,
}
}
export async function deleteVideos(videoIds) {
if (!videoIds || videoIds.length === 0) {
toast({
@ -827,7 +843,7 @@ export async function deleteDownload(downloadId) {
}
}
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag) {
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag};
if (mimetypes) {
body['mimetypes'] = mimetypes;
@ -850,6 +866,9 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
if (toYear) {
body['to_year'] = toYear;
}
if (order) {
body['order'] = order;
}
console.info('searching files', body);
const response = await apiPost(`${API_URI}/files/search`, body);
@ -1171,20 +1190,6 @@ export async function deleteTag(id, name) {
}
}
export async function fetchFile(path) {
const uri = `${API_URI}/files/file`;
const body = {file: path};
const response = await apiPost(uri, body);
if (response.ok) {
const content = await response.json();
return content['file'];
} else {
console.error('Unable to fetch file dict! See client logs.');
const message = await getErrorMessage(response, 'Unable to get File');
toast({type: 'error', title: 'File Error', description: message, time: 5000});
}
}
export async function sendNotification(message, url) {
const body = {message, url};
const response = await apiPost(`${API_URI}/notify`, body);

65
app/src/components/Archive.js
View file

@ -5,7 +5,6 @@ import {
CardHeader,
CardMeta,
Container,
Dropdown,
GridColumn,
GridRow,
Image,
@ -24,8 +23,8 @@ import {
ExternalCardLink,
FileIcon,
findPosterPath,
InfoHeader,
humanFileSize,
InfoHeader,
isoDatetimeToAgoPopup,
mimetypeColor,
PageContainer,
@ -39,7 +38,7 @@ import {
import {deleteArchives, postDownload, tagFileGroup, untagFileGroup} from "../api";
import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useArchive, useDomains, useSearchArchives, useSearchDomain, useSearchOrder} from "../hooks/customHooks";
import {useArchive, useDomains, useSearchArchives, useSearchOrder} from "../hooks/customHooks";
import {FileCards, FileRowTagIcon, FilesView} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import _ from "lodash";
@ -48,7 +47,7 @@ import {Button, Card, CardIcon, darkTheme, Header, Loader, Placeholder, Popup, S
import {SortableTable} from "./SortableTable";
import {taggedImageLabel, TagsSelector} from "../Tags";
import {toast} from "react-semantic-toasts-2";
import {Downloaders} from "./Vars";
import {API_ARCHIVE_UPLOAD_URI, Downloaders} from "./Vars";
function archiveFileLink(path, directory = false) {
if (path) {
@ -394,29 +393,47 @@ export function DomainsPage() {
</>;
}
export function SearchDomain() {
// A Dropdown which allows the user to filter by Archive domains.
const {domain, domains, setDomain} = useSearchDomain();
function ArchiveSettingsPage() {
const {t} = React.useContext(ThemeContext);
const handleChange = (e, {value}) => {
setDomain(value);
}
const urlClipboardButton = <APIButton
icon='copy'
onClick={() => navigator.clipboard.writeText(API_ARCHIVE_UPLOAD_URI)}
/>;
const dataFieldNameClipboardButton = <APIButton
icon='copy'
onClick={() => navigator.clipboard.writeText('singlefile_contents')}
/>;
const urlFieldNameClipboardButton = <APIButton
icon='copy'
onClick={() => navigator.clipboard.writeText('url')}
/>;
let domainOptions = [];
return <PageContainer>
<Header as='h1'>SingleFile Browser Extension</Header>
if (domains && domains.length > 0) {
domainOptions = domains.map(i => {
return {key: i['domain'], value: i['domain'], text: i['domain']}
});
}
return <>
<Dropdown selection search clearable fluid
placeholder='Domains'
options={domainOptions}
onChange={handleChange}
value={domain}
<p {...t}>
These are the settings necessary to configure the <a
href="https://github.com/gildas-lormeau/SingleFile?tab=readme-ov-file#install">SingleFile Browser
Extension</a> to automatically upload to your WROLPi.
</p>
<label {...t}>Upload URL</label>
<Input fluid
value={API_ARCHIVE_UPLOAD_URI}
label={urlClipboardButton}
/>
</>
<label {...t}>Data Field Name</label>
<Input fluid
value='singlefile_contents'
label={dataFieldNameClipboardButton}
/>
<label {...t}>URL Field Name</label>
<Input fluid
value='url'
label={urlFieldNameClipboardButton}
/>
</PageContainer>
}
function ArchivesPage() {
@ -599,12 +616,14 @@ export function ArchiveRoute() {
const links = [
{text: 'Archives', to: '/archive', end: true},
{text: 'Domains', to: '/archive/domains'},
{text: 'Settings', to: '/archive/settings'},
];
return <PageContainer>
<TabLinks links={links}/>
<Routes>
<Route path='/' element={<ArchivesPage/>}/>
<Route path='domains' element={<DomainsPage/>}/>
<Route path='settings' element={<ArchiveSettingsPage/>}/>
<Route path=':archiveId' element={<ArchivePage/>}/>
</Routes>
</PageContainer>

29
app/src/components/Common.js
View file

@ -10,6 +10,8 @@ import {
Container,
Dimmer,
DimmerDimmable,
GridColumn,
GridRow,
Icon as SIcon,
IconGroup,
Input,
@ -48,6 +50,7 @@ import {FilePreviewContext} from "./FilePreview";
import _ from "lodash";
import {killDownloads, startDownloads} from "../api";
import {allFrequencyOptions, NAME, semanticUIColorMap, validUrlRegex} from "./Vars";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
export function Paginator({activePage, onPageChange, totalPages, showFirstAndLast, size = 'mini'}) {
const handlePageChange = (e, {activePage}) => {
@ -1960,14 +1963,22 @@ export function getDistinctColor(hexColors) {
return hslToHex(newColor.h, newColor.s, newColor.l);
}
export const DismissibleMessage = ({storageName, children}) => {
const [dismissed, setDismissed] = useLocalStorage(storageName, false);
if (dismissed) {
return <React.Fragment/>
export const RefreshHeader = ({header, headerSize = 'h2', onRefresh, popupContents}) => {
const refreshButton = <APIButton icon='refresh' onClick={onRefresh}/>;
let popup;
if (popupContents) {
popup = <Popup
content={popupContents}
on='hover'
trigger={refreshButton}
/>;
}
return <Message onDismiss={() => setDismissed(true)}>
{children}
</Message>
return <Grid columns={2}>
<GridRow>
<GridColumn>
<Header as={headerSize}>{header}</Header>
</GridColumn>
<GridColumn textAlign='right'>{popup || refreshButton}</GridColumn>
</GridRow>
</Grid>
}

13
app/src/components/FilePreview.js
View file

@ -4,7 +4,7 @@ import {Grid, Header as SHeader, Image,} from "semantic-ui-react";
import {TagsSelector} from "../Tags";
import {Media} from "../contexts/contexts";
import {encodeMediaPath} from "./Common";
import {fetchFile, getFile, tagFileGroup, untagFileGroup} from "../api";
import {getFile, tagFileGroup, untagFileGroup} from "../api";
import {StlViewer} from "react-stl-viewer";
import {Button, Modal, ModalActions, ModalContent, ModalHeader} from "./Theme";
import {toast} from "react-semantic-toasts-2";
@ -200,7 +200,7 @@ export function FilePreviewProvider({children}) {
// Get the file again with its Tags.
const {path, primary_path} = previewFile;
try {
const file = await fetchFile(primary_path ?? path);
const file = await getFile(primary_path ?? path);
setPreviewFile(file);
} finally {
await handleCallbacks();
@ -310,6 +310,7 @@ export function FilePreviewProvider({children}) {
const lowerPath = path.toLowerCase();
const url = getMediaPathURL(previewFile);
const downloadURL = getDownloadPathURL(previewFile);
if (mimetype.startsWith('text/') && size > MAXIMUM_TEXT_SIZE) {
// Large text files should be downloaded.
window.open(downloadURL);
@ -336,6 +337,14 @@ export function FilePreviewProvider({children}) {
// No special handler for this file type, just open it.
setModalContent(getGenericPreviewModal(previewFile), url, downloadURL, path, taggable);
}
// Fetch the file so `FileGroup.viewed` is set.
try {
getFile(path);
} catch (e) {
console.error(e);
console.error('Failed to get file to set FileGroup.viewed');
}
}
}, [previewFile]);

1
app/src/components/Vars.js
View file

@ -12,6 +12,7 @@ export const FILES_MEDIA_URI = `https://${window.location.hostname}:${port}/medi
export const MAP_VIEWER_URI = `https://${window.location.hostname}:8084`;
export const ZIM_VIEWER_URI = `https://${window.location.hostname}:8085`;
export const HELP_VIEWER_URI = `https://${window.location.hostname}:8086`;
export const API_ARCHIVE_UPLOAD_URI = `http://${window.location.hostname}:8081/api/archive/upload`;
export let defaultFileOrder = '-published_datetime'; // Most recently published files first.
export let defaultSearchOrder = 'rank'; // Most relevant files first.

7
app/src/components/VideoPlayer.js
View file

@ -18,7 +18,7 @@ import {
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
import {VideoPlaceholder} from "./Placeholder";
import {useChannel} from "../hooks/customHooks";
import {useChannel, useVideoCaptions, useVideoExtras} from "../hooks/customHooks";
import {ThemeContext} from "../contexts/contexts";
import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme";
import {VideoCard} from "./Videos";
@ -192,6 +192,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
// Get the Video's channel, fallback to the URL's channel id.
const {channelId} = useParams();
const {channel} = useChannel(video && video.channel_id ? video.channel_id : channelId);
const {comments, captions} = useVideoExtras(videoFile?.video?.id);
if (videoFile === null) {
return <VideoPlaceholder/>;
@ -236,11 +237,11 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
const captionsPane = {
menuItem: 'Captions', render: () => <TabPane>
<pre>{video.caption || 'No captions available.'}</pre>
<pre>{captions || 'No captions available.'}</pre>
</TabPane>
};
const {poster_file, info_json_file, comments} = video;
const {poster_file, info_json_file} = video;
const filesPane = {
menuItem: 'Files', render: () => <TabPane>
<h3>Video File</h3>

27
app/src/components/admin/Settings.js
View file

@ -12,15 +12,15 @@ import {
Modal,
ModalContent,
ModalHeader,
Popup,
Segment
} from "../Theme";
import {ButtonGroup, Container, Dimmer, Dropdown, GridColumn, GridRow, Icon, Input} from "semantic-ui-react";
import {
APIButton,
ErrorMessage,
InfoPopup,
HotspotToggle,
InfoPopup,
RefreshHeader,
ThrottleToggle,
Toggle,
useMessageDismissal,
@ -489,16 +489,6 @@ export function SettingsPage() {
body = <ErrorMessage>Unable to fetch settings</ErrorMessage>
}
const refreshButton = <APIButton
icon='refresh'
onClick={fetchConfigs}
/>;
const refreshPopupButton = <Popup
content='Check if configs are valid'
on='hover'
trigger={refreshButton}
/>;
const controlSegment = <Segment>
<Header as='h3'>Control WROLPi</Header>
<HotspotToggle/>
@ -509,14 +499,11 @@ export function SettingsPage() {
</Segment>;
const configsSegment = <Segment>
<Grid columns={2}>
<GridRow>
<GridColumn>
<Header as='h2'>Configs</Header>
</GridColumn>
<GridColumn textAlign='right'>{refreshPopupButton}</GridColumn>
</GridRow>
</Grid>
<RefreshHeader
header='Refresh'
popupContents='Check if configs are valid'
onRefresh={fetchConfigs}
/>
<p>These configs control this WROLPi; they are the source of truth. Any changes to configs will be applied to
the database when imported (typically at startup).</p>

69
app/src/hooks/customHooks.js
View file

@ -18,6 +18,8 @@ import {
getStatistics,
getStatus,
getVideo,
getVideoCaptions,
getVideoComments,
getVideosStatistics,
postDumpConfig,
postImportConfig,
@ -423,6 +425,42 @@ export const useVideo = (videoId) => {
return {videoFile, prevFile, nextFile, fetchVideo};
}
export const useVideoExtras = (videoId) => {
// Fetches extra data to display on a Video's page.
const [comments, setComments] = useState(null);
const [captions, setCaptions] = useState(null);
const fetchComments = async () => {
try {
const result = await getVideoComments(videoId);
setComments(result.comments);
} catch (e) {
console.error(e);
}
}
const fetchCaptions = async () => {
try {
const result = await getVideoCaptions(videoId);
setCaptions(result.captions);
} catch (e) {
console.error(e);
}
}
useEffect(() => {
if (videoId) {
fetchComments();
fetchCaptions();
} else {
setComments(null);
setCaptions(null);
}
}, [videoId]);
return {comments, captions}
}
export const useChannel = (channel_id) => {
const emptyChannel = {
name: '',
@ -500,6 +538,37 @@ export const useChannels = () => {
return {channels, fetchChannels}
}
export const useSearchRecentFiles = () => {
const [searchFiles, setSearchFiles] = useState(null);
const [loading, setLoading] = useState(false);
const localSearchFiles = async () => {
setLoading(true);
try {
let [file_groups, total] = await filesSearch(
null, 12, null, null, null, [], false,
null, null, null, false, 'viewed');
setSearchFiles(file_groups);
} catch (e) {
console.error(e);
toast({
type: 'error',
title: 'Unexpected server response',
description: 'Could not get recent files',
time: 5000,
});
} finally {
setLoading(false);
}
}
React.useEffect(() => {
localSearchFiles();
}, []);
return {searchFiles, loading, fetchFiles: localSearchFiles};
}
export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => {
const {
activeTags, anyTag,

2
debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot
View file

@ -20,7 +20,7 @@ node --version
npm install
# Install Node console commands.
npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0
npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
# Install Python requirements.
python3 --version

2
docker-compose.yml
View file

@ -30,7 +30,7 @@ services:
- './alembic:/opt/wrolpi/alembic'
- './alembic.ini:/opt/wrolpi/alembic.ini'
ports:
- ${REACT_APP_API-127.0.0.1:8081}:8081
- ${REACT_APP_API-0.0.0.0:8081}:8081
user: '${UID-1000}:${GID-1000}'
healthcheck:
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]

20
docker/archive/Dockerfile
View file

@ -1,21 +1,21 @@
# Keep older singlefile version so we can keep screenshots at specific ratio.
FROM capsulecode/singlefile:v0.3.32
FROM browserless/chrome:1.61.1-chrome-stable
USER root
WORKDIR /app
# Install Python for main.py
ENV PYTHONUNBUFFERED=1
RUN apk add --update --no-cache python3 python3-dev curl && ln -sf python3 /usr/bin/python
RUN python3 -m ensurepip
RUN pip3 install --no-cache --upgrade pip setuptools
RUN apt update && apt install -y python3 python3-dev python3-venv curl && ln -sf python3 /usr/bin/python
RUN python3 -m venv /app/venv
RUN . /app/venv/bin/activate
# Install a newer singlefile and Readability.
RUN npm install -g single-file-cli@2.0.73 readability-extractor@0.0.6
# Install the Python wrapper and it's requirements.
COPY docker/archive/requirements.txt /app/requirements.txt
RUN pip3 install -r /app/requirements.txt
RUN /app/venv/bin/pip3 install -r /app/requirements.txt
COPY docker/archive /app
# Install a newer singlefile and Readability.
RUN npm install -g single-file-cli@1.1.54 readability-extractor@0.0.6
ENTRYPOINT [ "python3", "/app/main.py"]
# Launch Archive app in the virtual environment.
ENTRYPOINT [ "/app/venv/bin/python3", "/app/main.py"]

17
docker/archive/main.py
View file

@ -28,6 +28,8 @@ logger.addHandler(ch)
SINGLEFILE_PATH = pathlib.Path('/usr/src/app/node_modules/single-file-cli/single-file')
if not SINGLEFILE_PATH.is_file():
SINGLEFILE_PATH = pathlib.Path('/usr/src/app/node_modules/single-file/cli/single-file')
if not SINGLEFILE_PATH.is_file():
SINGLEFILE_PATH = pathlib.Path('/usr/src/.nvm/versions/node/v18.19.0/bin/single-file')
if not SINGLEFILE_PATH.is_file():
raise FileNotFoundError("Can't find single-file executable!")
@ -77,8 +79,6 @@ async def call_single_file(url) -> bytes:
"""
logger.info(f'archiving {url}')
cmd = f'{SINGLEFILE_PATH}' \
r' --browser-executable-path /usr/bin/chromium-browser' \
r' --browser-args [\"--no-sandbox\"]' \
r' --dump-content ' \
f' "{url}"'
logger.debug(f'archive cmd: {cmd}')
@ -86,7 +86,7 @@ async def call_single_file(url) -> bytes:
if return_code != 0 or not stdout:
if stderr:
for line in stderr.splitlines():
logger.error(line)
logger.error(line.decode())
raise RuntimeError(f'Failed to single-file {url} got the following error: {stderr}')
logger.debug(f'done archiving for {url}')
return stdout
@ -112,7 +112,7 @@ async def extract_readability(path: str, url: str) -> dict:
async def take_screenshot(url: str) -> bytes:
cmd = '/usr/bin/chromium-browser' \
cmd = '/usr/bin/google-chrome' \
' --headless' \
' --disable-gpu' \
' --no-sandbox' \
@ -153,8 +153,15 @@ def prepare_bytes(b: bytes) -> str:
@app.post('/json')
async def post_archive(request: Request):
url = request.json['url']
try:
singlefile = await call_single_file(url)
# May have been passed the singlefile contents, do the extractions and return.
singlefile = request.json.get('singlefile')
singlefile = base64.b64decode(singlefile) if singlefile else None
# Use provided singlefile, or fetch and create from the internet.
singlefile = singlefile or await call_single_file(url)
# Use html suffix so chrome screenshot recognizes it as an HTML file.
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
fh.write(singlefile)

2
docker/archive/requirements.txt
View file

@ -1 +1 @@
sanic==23.3.0
sanic==24.12.0

139
modules/archive/__init__.py
View file

@ -1,18 +1,15 @@
import asyncio
import json
import pathlib
import tempfile
from abc import ABC
from json import JSONDecodeError
from typing import List, Tuple, Iterable
from selenium import webdriver
from sqlalchemy import not_
from sqlalchemy.orm import Session
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
slow_logger, html_screenshot, get_title_from_html
slow_logger, get_title_from_html
from wrolpi.db import optional_session, get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
@ -38,6 +35,11 @@ class ArchiveDownloader(Downloader, ABC):
def __repr__(self):
return f'<ArchiveDownloader>'
@optional_session
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
return file_groups
async def do_download(self, download: Download) -> DownloadResult:
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
@ -45,12 +47,13 @@ class ArchiveDownloader(Downloader, ABC):
if DOCKERIZED or PYTEST:
# Perform the archive in the Archive docker container. (Typically in the development environment).
singlefile, readability, screenshot = await request_archive(download.url)
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
archive_id = archive.id
else:
# Perform the archive using locally installed executables.
singlefile, readability, screenshot = await self.do_archive(download)
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
archive_id = archive.id
singlefile = await self.do_singlefile(download)
archive = await lib.singlefile_to_archive(singlefile)
archive_id = archive.id
with get_db_session() as session:
archive = Archive.find_by_id(archive_id, session)
@ -67,12 +70,7 @@ class ArchiveDownloader(Downloader, ABC):
return DownloadResult(success=True, location=f'/archive/{archive.id}')
@optional_session
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
return file_groups
async def do_singlefile(self, download: Download) -> Tuple[bytes, dict]:
async def do_singlefile(self, download: Download) -> bytes:
"""Create a Singlefile from the archive's URL."""
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
str(SINGLE_FILE_BIN),
@ -83,115 +81,26 @@ class ArchiveDownloader(Downloader, ABC):
'--load-deferred-images-dispatch-scroll-event',
download.url,
)
return_code, logs, stdout = await self.process_runner(
download.id,
download.url,
cmd,
pathlib.Path('/home/wrolpi'),
)
cwd = pathlib.Path('/home/wrolpi')
cwd = cwd if cwd.is_dir() else None
result = await self.process_runner(download, cmd, cwd)
stderr = logs['stderr'].decode()
log_output = stderr or stdout.decode() or 'No stderr or stdout!'
stderr = result.stderr.decode()
log_output = stderr or result.stdout.decode() or 'No stderr or stdout!'
if return_code != 0:
if result.return_code != 0:
e = ChildProcessError(log_output[:1000])
raise RuntimeError(f'singlefile exited with {return_code}') from e
raise RuntimeError(f'singlefile exited with {result.return_code}') from e
if not stdout:
if not result.stdout:
e = ChildProcessError(log_output[:1000])
raise RuntimeError(f'Singlefile created was empty: {download.url}') from e
if SINGLEFILE_HEADER.encode() not in stdout:
if SINGLEFILE_HEADER.encode() not in result.stdout:
e = ChildProcessError(log_output[:1000])
raise RuntimeError(f'Singlefile created was invalid: {download.url}') from e
return stdout, logs
async def do_readability(self, download: Download, html: bytes) -> dict:
"""Extract the readability dict from the provided HTML."""
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
fh.write(html)
cmd = (READABILITY_BIN, fh.name, download.url)
logger.debug(f'readability cmd: {cmd}')
return_code, logs, stdout = await self.process_runner(
download.id,
download.url,
cmd,
pathlib.Path('/home/wrolpi'),
)
if return_code == 0:
if not stdout:
raise RuntimeError('readability stdout was empty')
try:
readability = json.loads(stdout)
except TypeError or JSONDecodeError:
# JSON was invalid.
e = ChildProcessError(stdout.decode() if stdout else 'No stdout')
if logs and (stderr := logs.get('stderr')):
e = ChildProcessError(stderr.decode())
raise RuntimeError(f'Failed to extract readability from {download.url}') from e
logger.debug(f'done readability for {download.url}')
return readability
else:
logger.error(f'Failed to extract readability for {download.url}')
e = ChildProcessError(stdout.decode() if stdout else 'No stdout')
if logs and (stderr := logs.get('stderr')):
e = ChildProcessError(stderr.decode())
raise RuntimeError(f'Failed to extract readability for {download.url} got {return_code}') from e
@staticmethod
async def do_screenshot_url(download: Download) -> bytes:
"""Use Chromium to get a screenshot of the Download's URL."""
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of
# the page.
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument('disable-gpu')
options.add_argument('window-size=1280x720')
driver = webdriver.Chrome(chrome_options=options)
driver.get(download.url)
screenshot = driver.get_screenshot_as_png()
return screenshot
async def do_archive(self, download: Download) -> Tuple[bytes, dict, bytes]:
"""Use locally installed executables to create an Archive.
Creates a Singlefile, readability file, and screenshot file.
@warning: Will not raise errors if readability or screenshot cannot be extracted.
"""
singlefile, logs = await self.do_singlefile(download)
# Extract Readability from the Singlefile.
try:
readability = await self.do_readability(download, singlefile)
except RuntimeError:
# Readability is not required.
readability = None
screenshot = b''
try:
# Screenshot the Singlefile first.
screenshot = html_screenshot(singlefile)
except Exception as e:
logger.error(f'Failed to screenshot file for {download.url}', exc_info=e)
if not screenshot:
# Screenshot of singlefile failed. Download the URL again.
try:
screenshot = await self.do_screenshot_url(download)
except Exception as e:
# Screenshot failed.
logger.error(f'Failed to screenshot {download.url}', exc_info=e)
singlefile_len = len(singlefile) if singlefile else None
readability_len = len(readability) if readability else None
screenshot_len = len(screenshot) if screenshot else None
logger.debug(f'do_archive of {download.url} finished: {singlefile_len=} {readability_len=} {screenshot_len=}')
return singlefile, readability, screenshot
return result.stdout
archive_downloader = ArchiveDownloader()
@ -258,7 +167,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
archive.validate()
archive.flush()
file_group.title = file_group.a_text = title
file_group.title = file_group.a_text = title or archive.file_group.title
file_group.d_text = contents
file_group.data = {
'id': archive.id,

39
modules/archive/api.py
View file

@ -7,7 +7,9 @@ from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
from wrolpi.switches import register_switch_handler
from . import lib, schema
NAME = 'archive'
@ -29,7 +31,7 @@ async def get_archive(_: Request, archive_id: int):
@archive_bp.delete('/<archive_ids:int>', name='archive_delete_one')
@archive_bp.delete('/<archive_ids:[0-9,]+>', name='archive_delete_many')
@openapi.description('Delete an individual archive')
@openapi.description('Delete Archives')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@wrol_mode_check
async def delete_archive(_: Request, archive_ids: str):
@ -70,3 +72,38 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
body.headline)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)
@register_switch_handler('singlefile_upload_switch_handler')
async def singlefile_upload_switch_handler(singlefile):
archive = await lib.singlefile_to_archive(singlefile)
logger.info(f'Created Archive from upload: {archive}')
name = archive.file_group.title or archive.file_group.url
Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location)
@archive_bp.get('/upload')
@openapi.definition(
summary='A message to confirm to the user that they have the correct upload URL.'
)
async def get_upload_singlefile(request: Request):
return response.text('This is the URL to upload files using the SingleFile browser extension.'
' This requires a POST request.')
@archive_bp.post('/upload')
@openapi.definition(
summary='Upload SingleFile from SingleFile browser extension and convert it to an Archive.'
)
async def post_upload_singlefile(request: Request):
url = request.form['url'][0]
singlefile = request.files['singlefile_contents'][0].body
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}')
# Get the URL to ensure that the singlefile is valid.
lib.get_url_from_singlefile(singlefile)
# Send processing to background task so the extension can continue.
singlefile_upload_switch_handler.activate_switch(dict(singlefile=singlefile))
# Return empty json response because SingleFile extension expects a JSON response.
return json_response(dict(), status=HTTPStatus.OK)

105
modules/archive/lib.py
View file

@ -1,10 +1,15 @@
import base64
import base64
import gzip
import json
import os
import pathlib
import re
import shlex
import tempfile
from dataclasses import dataclass
from datetime import datetime
from json import JSONDecodeError
from typing import Optional, Tuple, List, Union
from urllib.parse import urlparse
@ -14,13 +19,14 @@ from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi import dates
from wrolpi.cmd import READABILITY_BIN, run_command
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config, html_screenshot
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
from wrolpi.tags import tag_append_sub_select_where
from wrolpi.vars import PYTEST
from wrolpi.vars import PYTEST, DOCKERIZED
logger = logger.getChild(__name__)
@ -106,11 +112,11 @@ def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles:
ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response.
async def request_archive(url: str) -> Tuple[str, Optional[str], Optional[str]]:
async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Optional[dict], Optional[str]]:
"""Send a request to the archive service to archive the URL."""
logger.info(f'Sending archive request to archive service: {url}')
data = {'url': url}
data = dict(url=url, singlefile=singlefile)
try:
async with aiohttp_post(f'{ARCHIVE_SERVICE}/json', json_=data, timeout=ARCHIVE_TIMEOUT) as response:
status = response.status
@ -229,9 +235,9 @@ def get_or_create_domain(session: Session, url) -> Domain:
SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
def get_url_from_singlefile(html: bytes) -> str:
def get_url_from_singlefile(html: bytes | str) -> str:
"""Extract URL from SingleFile contents."""
html = html.decode()
html = html.decode() if isinstance(html, bytes) else html
if SINGLEFILE_HEADER not in html:
raise RuntimeError(f'Not a singlefile!')
@ -313,7 +319,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
except json.decoder.JSONDecodeError:
# Was not valid JSON.
schema = None
if isinstance(schema, dict) and schema.get('@context') in ('https://schema.org', 'http://schema.org'):
if isinstance(schema, dict) and '://schema.org' in schema.get('@context'):
# Found https://schema.org/
if headline := schema.get('headline'):
metadata.title = metadata.title or headline
@ -347,6 +353,17 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
metadata.author = author
else:
logger.warning(f'Unable to parse author schema: {author}')
elif authors := schema.get('authors'):
# Use the first Author.
author = authors[0]
if isinstance(author, dict):
author = author.get('name') or author
if isinstance(author, str):
metadata.author = author
else:
logger.warning(f'Unable to parse author schema: {author}')
# Assume UTC if no timezone.
if metadata.published_datetime and not metadata.published_datetime.tzinfo and assume_utc:
@ -368,7 +385,7 @@ def get_archive(session, archive_id: int) -> Archive:
def delete_archives(*archive_ids: List[int]):
"""Delete an Archive and all of it's files."""
"""Delete an Archive and all of its files."""
with get_db_session(commit=True) as session:
archives: List[Archive] = list(session.query(Archive).filter(Archive.id.in_(archive_ids)))
if not archives:
@ -535,3 +552,75 @@ async def search_domains_by_name(name: str, limit: int = 5, session: Session = N
.limit(limit) \
.all()
return domains
async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
"""Extract the readability dict from the provided HTML."""
with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file:
singlefile_file.write(html.encode() if isinstance(html, str) else html)
singlefile_file.flush()
os.fsync(singlefile_file.fileno())
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
READABILITY_BIN, singlefile_file.name, shlex.quote(url))
# Docker containers may not have this directory. But, this directory is necessary on RPi.
cwd = '/home/wrolpi' if os.path.isdir('/home/wrolpi') else None
result = await run_command(cmd, cwd=cwd, timeout=timeout, debug=True)
logger.debug(f'readability for {url} exited with {result.return_code}')
stdout, stderr = result.stdout.decode(), result.stderr.decode()
if result.return_code == 0:
if not stdout:
raise RuntimeError('readability stdout was empty')
try:
readability = json.loads(stdout)
except TypeError or JSONDecodeError:
# JSON was invalid.
e = ChildProcessError(stdout if stdout else 'No stdout')
if stderr:
e = ChildProcessError(stderr)
raise RuntimeError(f'Failed to extract readability from {url}') from e
logger.debug(f'done readability for {url}')
return readability
else:
logger.error(f'Failed to extract readability for {url}')
e = ChildProcessError(stdout if stdout else 'No stdout')
if stderr:
e = ChildProcessError(stderr)
raise RuntimeError(f'Failed to extract readability for {url} got {result.return_code}') from e
async def singlefile_to_archive(singlefile: bytes) -> Archive:
"""
Convert a SingleFile to an Archive.
This is done by extracting readability, creating a screenshot, then attaching them to an Archive/FileGroup.
"""
# Get URL first because it does some simple checking of `singlefile`
url = get_url_from_singlefile(singlefile)
singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile
if DOCKERIZED:
# Perform the archive in the Archive docker container. (Typically in the development environment).
singlefile = base64.b64encode(singlefile).decode()
logger.debug(f'singlefile_to_archive sending to archive service: {url}')
singlefile, readability, screenshot = await request_archive(url, singlefile=singlefile)
else:
# JSON from readability-extractor
readability = dict()
try:
logger.debug(f'singlefile_to_archive extracting readability: {url}')
readability = await html_to_readability(singlefile, url)
except RuntimeError as e:
logger.error(f'Failed to extract readability from: {url}', exc_info=e)
screenshot = None
try:
logger.debug(f'singlefile_to_archive creating screenshot: {url}')
screenshot = html_screenshot(singlefile)
except Exception as e:
logger.error(f'Failed to extract screenshot from: {url}', exc_info=e)
logger.trace(f'singlefile_to_archive modeling: {url}')
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot)
return archive

5
modules/archive/models.py
View file

@ -318,6 +318,11 @@ class Archive(Base, ModelHelper):
def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile:
return self.file_group.add_tag(tag_id_or_name, session)
@property
def location(self):
"""The location where this Archive can be viewed in the UI."""
return f'/archive/{self.id}'
class Domain(Base, ModelHelper):
__tablename__ = 'domains' # plural to avoid conflict

31
modules/archive/test/test_api.py
View file

@ -3,7 +3,8 @@ from http import HTTPStatus
import pytest
from modules.archive import lib
from modules.archive import lib, Archive
from wrolpi.test.common import skip_circleci
def check_results(test_client, data, ids):
@ -227,3 +228,31 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'] == []
@skip_circleci
@pytest.mark.asyncio
async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form,
await_switches, events_history):
"""Test converting a SingleFile from the SingleFile browser extension to an Archive."""
singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url')
forms = [
dict(name='url', value='https://example.com/form-url'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents')
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
archive, = test_session.query(Archive).all()
assert archive.singlefile_path.is_file(), 'Singlefile should have been saved.'
assert archive.readability_path.is_file(), 'Readability should be extracted.'
assert archive.readability_json_path.is_file(), 'Readability json should be extracted.'
assert archive.screenshot_path.is_file(), 'Screenshot should have been created'
assert archive.file_group.title == 'upload title', 'Title was not correct'
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
events = list(events_history)
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'

29
modules/archive/test/test_lib.py
View file

@ -561,6 +561,35 @@ async def test_refresh_archives_index(test_session, make_files_structure):
assert archive.file_group.indexed is True
@pytest.mark.asyncio
async def test_archive_meta(async_client, test_session, make_files_structure):
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en">
<script data-vue-meta="ssr" type="application/ld+json">
{"@context":"https://schema.org/","@type":"NewsArticle","headline":"The Headline","description":"The Description","authors":[{"name":"A.B.C.","@type":"Person"}],"datePublished":"2021-06-20T10:00"}
</script>
</html>'''
singlefile, *_ = make_files_structure({
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
'archive/example.com/2021-10-05-16-20-10_NA.png': None,
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents',
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
})
await files_lib.refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
assert archive.file_group.author == 'A.B.C.'
assert archive.file_group.published_datetime == datetime(2021, 6, 20, 10, 0, tzinfo=pytz.utc)
assert archive.file_group.title == 'The Headline'
assert archive.file_group.b_text == 'The Description'
assert archive.file_group.indexed is True
@pytest.mark.asyncio
async def test_refresh_archives_deleted_singlefile(test_session, make_files_structure, singlefile_contents_factory):
"""Removing a Singlefile file from a FileGroup makes that group no longer an Archive."""

1
modules/inventory/__init__.py
View file

@ -1,4 +1,3 @@
import wrolpi
from wrolpi.db import get_db_session
from .api import inventory_bp
from .inventory import logger, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES

30
modules/map/lib.py
View file

@ -1,5 +1,5 @@
import asyncio
import subprocess
from datetime import timedelta
from pathlib import Path
from typing import List
@ -8,9 +8,9 @@ from sqlalchemy.orm import Session
from modules.map.models import MapFile
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import SUDO_BIN
from wrolpi.cmd import SUDO_BIN, run_command
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp
from wrolpi.db import optional_session, get_db_session
from wrolpi.events import Events
from wrolpi.vars import PROJECT_DIR, IS_RPI5
@ -37,7 +37,7 @@ def is_pbf_file(pbf: Path) -> bool:
def is_dump_file(path: Path) -> bool:
"""Uses file command to check type of a file. Returns True if a file is a Postgresql dump file."""
"""Uses file command to check the type of file. Returns True if a file is a Postgresql dump file."""
cmd = ('/usr/bin/file', path)
try:
output = subprocess.check_output(cmd)
@ -188,24 +188,22 @@ async def run_import_command(*paths: Path) -> int:
paths = ' '.join(str(i) for i in paths)
# Run with sudo so renderd can be restarted.
cmd = f'{SUDO_BIN} {PROJECT_DIR}/scripts/import_map.sh {paths}'
cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', paths)
import_logger.warning(f'Running map import command: {cmd}')
start = now()
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
result = await run_command(cmd, debug=True)
stdout, stderr = await proc.communicate()
elapsed = now() - start
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
import_logger.info(f'{success} import of {repr(paths)} took {timedelta_to_timestamp(elapsed)}')
if proc.returncode != 0:
success = 'Successful' if result.return_code == 0 else 'Unsuccessful'
import_logger.info(
f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}')
if result.return_code != 0:
# Log all lines. Truncate long lines.
for line in stdout.decode().splitlines():
for line in result.stdout.decode().splitlines():
import_logger.debug(line[:500])
for line in stderr.decode().splitlines():
for line in result.stderr.decode().splitlines():
import_logger.error(line[:500])
raise ValueError(f'Importing map file failed with return code {proc.returncode}')
raise ValueError(f'Importing map file failed with return code {result.return_code}')
return int(elapsed.total_seconds())
return result.elapsed
@optional_session

54
modules/map/test/test_api.py
View file

@ -16,7 +16,7 @@ def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK):
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_status_and_import(test_client, test_session, make_files_structure, mock_create_subprocess_shell):
def test_status_and_import(test_client, test_session, make_files_structure, mock_run_command):
"""PBF files can be imported, and the status of the import can be monitored."""
pbf1, pbf2, dump1 = make_files_structure([
'map/pbf/some-country.osm.pbf',
@ -32,11 +32,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
@ -45,11 +43,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
@ -58,11 +54,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': True, 'path': 'map/dump/dumpy.dump'},
@ -74,7 +68,7 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_create_subprocess_shell):
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_run_command):
"""Multiple PBFs can be imported. Importing a second time overwrites the previous imports."""
pbf1, pbf2, pbf3, dump = make_files_structure([
'map/pbf/country1.osm.pbf',
@ -83,11 +77,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
'map/dumpy.dump',
])
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf2), str(pbf1)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf2), str(pbf1)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Two PBFs can be merged and imported.
expected = [
@ -98,11 +90,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf3), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf3), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# New imports overwrite old imports.
expected = [
@ -113,12 +103,10 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Dump does not overwrite.
expected = [

18
modules/map/test/test_lib.py
View file

@ -22,7 +22,7 @@ def test_get_map_paths(test_directory, make_files_structure):
@pytest.mark.asyncio
async def test_run_import_command(test_directory, mock_create_subprocess_shell):
async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result):
"""
Import map files. Files to import are checked for validity before importing. Any errors returned by
asyncio.create_subprocess_shell are caught and reported.
@ -38,22 +38,14 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell):
dump_file = test_directory / 'dumpy.dump'
dump_file.touch()
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it succeeds.
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
communicate_return=(b'out', b'error')
)
mock_is_pbf_file.return_value = True
await lib.run_import_command(pbf_file)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it fails.
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
communicate_return=(b'out', b'error'),
return_code=1,
)
mock_run_command(run_command_bad_result)
mock_is_pbf_file.return_value = True
with pytest.raises(ValueError):
await lib.run_import_command(pbf_file)
@ -102,7 +94,7 @@ def test_seconds_to_import_rpi5(size, expected):
assert lib.seconds_to_import(size, True) == expected
def test_get_custom_map_directory(test_directory, test_wrolpi_config):
def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config):
"""Custom directory can be used for map directory."""
# Default location.
assert lib.get_map_directory() == (test_directory / 'map')

1
modules/videos/channel/api.py
View file

@ -9,7 +9,6 @@ from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, get_media_directory, get_relative_to_media_directory
from wrolpi.downloader import download_manager
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
from wrolpi.vars import PYTEST

2
modules/videos/channel/lib.py
View file

@ -14,7 +14,7 @@ from wrolpi.errors import APIError, ValidationError, RefreshConflict
from wrolpi.vars import PYTEST
from .. import schema
from ..common import check_for_channel_conflicts
from ..errors import UnknownChannel, ChannelDirectoryConflict
from ..errors import UnknownChannel
from ..lib import save_channels_config
from ..models import Channel, Video

5
modules/videos/channel/test/test_api.py
View file

@ -57,7 +57,8 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
@pytest.mark.asyncio
async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel, test_download_manager):
async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel,
test_download_manager):
"""A channel does not require a download frequency."""
# No downloads are scheduled.
assert len(test_download_manager.get_downloads(test_session)) == 0
@ -465,7 +466,7 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
# Create recurring download which uses the Channel's directory.
download = test_download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
destination=str(test_directory / 'videos/Channel Name'))
destination=str(test_directory / 'videos/Channel Name'))
test_session.commit()
save_channels_config()
assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name')

37
modules/videos/common.py
View file

@ -1,9 +1,7 @@
import asyncio
import json
import os
import pathlib
import re
import shlex
import subprocess
import tempfile
from datetime import timedelta
@ -16,7 +14,7 @@ from PIL import Image
from sqlalchemy.orm import Session
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import FFPROBE_BIN
from wrolpi.cmd import FFPROBE_BIN, run_command
from wrolpi.common import logger, get_media_directory
from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
@ -213,28 +211,23 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict:
],
}
"""
cmd = f'{FFPROBE_BIN}' \
f' -print_format json' \
f' -loglevel quiet' \
f' -show_streams' \
f' -show_format' \
f' -show_chapters' \
f' {shlex.quote(str(video_path.absolute()))}'
cmd = (FFPROBE_BIN,
'-print_format', 'json',
'-loglevel', 'quiet',
'-show_streams',
'-show_format',
'-show_chapters',
video_path.absolute())
result = await run_command(cmd, timeout=60)
if result.return_code != 0:
raise RuntimeError(f'Got non-zero exit code: {result.return_code}')
try:
proc = await asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = await proc.communicate()
content = json.loads(result.stdout.decode().strip())
except Exception as e:
logger.error(f'Failed to run ffprobe json', exc_info=e)
raise
if proc.returncode != 0:
raise RuntimeError(f'Got non-zero exit code: {proc.returncode}')
try:
content = json.loads(stdout.decode().strip())
except Exception as e:
logger.debug(stdout.decode())
logger.debug(result.stdout.decode())
logger.error(f'Failed to load ffprobe json', exc_info=e)
raise

17
modules/videos/conftest.py
View file

@ -2,7 +2,6 @@ import json
import pathlib
import shutil
from http import HTTPStatus
from itertools import zip_longest
from typing import List
from uuid import uuid4
@ -14,6 +13,7 @@ from modules.videos.downloader import VideoDownloader, ChannelDownloader
from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination
from modules.videos.models import Channel, Video
from wrolpi.api_utils import api_app
from wrolpi.cmd import CommandResult
from wrolpi.downloader import DownloadFrequency, DownloadManager
from wrolpi.files.models import FileGroup, Directory
from wrolpi.vars import PROJECT_DIR
@ -208,22 +208,11 @@ def mock_video_prepare_filename():
@pytest.fixture
def mock_video_process_runner():
with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_process_runner.return_value = (0, {'stdout': None, 'stderr': None}, None)
mock_process_runner.return_value = \
CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
yield mock_process_runner
@pytest.fixture
def assert_video_ids(test_session):
"""Check that only the expected Videos are in the DB."""
def checker(expected_video_ids: List[int]):
video_ids = [i.id for i in test_session.query(Video).order_by(Video.id)]
for id_, expected in zip_longest(video_ids, expected_video_ids):
assert id_ == expected, f'Video ids were not as expected: {expected_video_ids=} != {video_ids=}'
return checker
@pytest.fixture
def video_with_search_factory(test_session, test_directory, video_file_factory):
"""A factory that creates a Video record with an associated video File record."""

11
modules/videos/downloader.py
View file

@ -416,13 +416,12 @@ class VideoDownloader(Downloader, ABC):
'-o', video_path,
url,
)
return_code, logs, _ = await self.process_runner(download_id, url, cmd, out_dir)
result = await self.process_runner(download, cmd, out_dir)
stdout = result.stdout.decode()
stderr = result.stderr.decode()
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
if return_code != 0:
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {return_code}'
if result.return_code != 0:
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}'
return DownloadResult(
success=False,
error=error,

6
modules/videos/models.py
View file

@ -65,19 +65,13 @@ class Video(ModelHelper, Base):
except Exception as e:
logger.error(f'{self} ffprobe_json is invalid', exc_info=e)
# TODO these are large objects. Can they be fetched on demand?
captions = self.file_group.d_text
comments = self.get_comments()
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
d['video'] = dict(
caption=captions,
caption_files=self.caption_files,
channel=channel,
channel_id=self.channel_id,
codec_names=codec_names,
codec_types=codec_types,
comments=comments,
comments_failed=self.comments_failed,
description=self.file_group.c_text or self.get_video_description(),
have_comments=self.have_comments,

10
modules/videos/schema.py
View file

@ -155,6 +155,16 @@ class VideoResponse:
next = ExtendedVideo
@dataclass
class VideoCommentsResponse:
comments: list[dict]
@dataclass
class VideoCaptionsResponse:
captions: str
@dataclass
class VideoSearchRequest:
search_str: Optional[str] = None

4
modules/videos/test/test_downloader.py
View file

@ -296,10 +296,10 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i
await video_download_manager.wait_for_all_downloads()
mock_video_process_runner.assert_called_once()
download_id, video_url, _, out_dir = mock_video_process_runner.call_args[0]
download, cmd, out_dir = mock_video_process_runner.call_args[0]
download: Download = test_session.query(Download).one()
assert video_url == download.url
assert download.url in cmd
assert test_session.query(Channel).one()
video: Video = test_session.query(Video).one()

18
modules/videos/video/api.py
View file

@ -28,6 +28,24 @@ def video_get(_: Request, video_id: int):
return json_response({'file_group': video, 'prev': previous_video, 'next': next_video})
@video_bp.get('/video/<video_id:int>/comments')
@openapi.description('Get Video comments')
@openapi.response(HTTPStatus.OK, schema.VideoCommentsResponse)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
def video_get_comments(_: Request, video_id: int):
video = lib.get_video(video_id)
return json_response({'comments': video.get_comments()})
@video_bp.get('/video/<video_id:int>/captions')
@openapi.description('Get Video captions')
@openapi.response(HTTPStatus.OK, schema.VideoCaptionsResponse)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
def video_get_captions(_: Request, video_id: int):
video = lib.get_video(video_id)
return json_response({'captions': video.file_group.d_text})
@video_bp.post('/search')
@openapi.definition(
summary='Search Video titles and captions',

14
modules/videos/video/lib.py
View file

@ -1,14 +1,14 @@
import asyncio
import random
from datetime import timedelta
from typing import Tuple, Optional, List, Dict
from typing import Tuple, Optional, List
import yt_dlp
from sqlalchemy import or_
from sqlalchemy.orm import Session
from modules.videos.models import Video
from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check
from wrolpi.common import logger, limit_concurrent, wrol_mode_check
from wrolpi.dates import now
from wrolpi.db import get_db_session, optional_session
from wrolpi.downloader import download_manager
@ -17,7 +17,6 @@ from wrolpi.files.models import FileGroup
from wrolpi.tags import tag_append_sub_select_where
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR
from ..errors import UnknownVideo
from ..lib import save_channels_config
logger.getChild(__name__)
@ -38,6 +37,15 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
return video, previous_video, next_video
def get_video(video_id: int) -> Video:
"""
Get a Video, with it's prev/next videos. Mark the Video as viewed.
"""
with get_db_session() as session:
video = Video.find_by_id(video_id, session=session)
return video
VIDEO_ORDERS = {
'published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) ASC, LOWER(fg.primary_path) ASC',
'-published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST,'

20
modules/videos/video/test/test_api.py
View file

@ -187,3 +187,23 @@ async def test_wrol_mode(async_client, simple_channel, simple_video, wrol_mode_f
await wrol_mode_fixture(False)
assert not test_download_manager.stopped.is_set()
@pytest.mark.asyncio
async def test_api_video_extras(async_client, simple_channel, video_factory):
"""Can fetch extra data about a video (comments/captions)."""
info_json = {'duration': 5, 'epoch': 12345, 'comments': [{'some': 'comments'}]}
video = video_factory(simple_channel.id, with_info_json=info_json, with_caption_file=True)
request, response = await async_client.get(f'/api/videos/video/{video.id}/captions')
assert response.status_code == HTTPStatus.OK
assert response.json.get('captions')
request, response = await async_client.get(f'/api/videos/video/{video.id}/comments')
assert response.status_code == HTTPStatus.OK
assert response.json.get('comments')
request, response = await async_client.get('/api/videos/video/123/captions')
assert response.status_code == HTTPStatus.NOT_FOUND
request, response = await async_client.get('/api/videos/video/123/comments')
assert response.status_code == HTTPStatus.NOT_FOUND

2
modules/zim/downloader.py
View file

@ -74,7 +74,7 @@ class KiwixZimDownloader(Downloader):
zim_directory = lib.get_zim_directory()
zim_directory.mkdir(parents=True, exist_ok=True)
output_path = await self.download_file(download.id, download.url, zim_directory)
output_path = await self.download_file(download, download.url, zim_directory)
# Notify the maintainer if outdated Zim files are lying around.
lib.flag_outdated_zim_files()

27
modules/zim/lib.py
View file

@ -15,6 +15,7 @@ from modules.zim.errors import UnknownZim, UnknownZimSubscription
from modules.zim.kiwix import KIWIX_CATALOG
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
from wrolpi import flags
from wrolpi.cmd import run_command
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate
from wrolpi.db import get_db_session, optional_session, get_db_curs
@ -471,13 +472,12 @@ async def check_zim(path: pathlib.Path) -> int:
@warning: Only performs the checksum check!
"""
cmd = f'zimcheck -C {path.absolute()}'
proc = await asyncio.create_subprocess_shell(cmd)
stdout, stderr = await proc.communicate()
logger.debug(f'zimcheck returned {proc.returncode}')
if proc.returncode != 0 and stderr:
logger.debug(stderr)
return proc.returncode
cmd = ['zimcheck', '-C', path.absolute()]
result = await run_command(cmd)
logger.debug(f'zimcheck returned {result.return_code}')
if result.return_code != 0 and result.stderr:
logger.debug(result.stderr)
return result.return_code
ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim')
@ -566,10 +566,9 @@ async def restart_kiwix():
logger.info('Restarting Kiwix serve')
cmd = f'sudo /usr/bin/systemctl restart wrolpi-kiwix.service'
proc = await asyncio.create_subprocess_shell(cmd)
stdout, stderr = await proc.communicate()
logger.debug(f'systemctl returned {proc.returncode}')
if proc.returncode != 0 and stderr:
logger.debug(stderr)
return proc.returncode
cmd = ('sudo', '/usr/bin/systemctl', 'restart' 'wrolpi-kiwix.service')
result = await run_command(cmd)
logger.debug(f'systemctl returned {result.return_code}')
if result.return_code != 0 and result.stderr:
logger.debug(result.stderr)
return result.return_code

2
modules/zim/test/test_lib.py
View file

@ -12,6 +12,7 @@ from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, save_downloads_config
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_circleci
@pytest.mark.asyncio
@ -182,6 +183,7 @@ def test_zim_all_entries(test_session, test_zim):
assert len(entries) == 12
@skip_circleci
@pytest.mark.asyncio
async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock,
await_switches):

2
pi-gen/stage2/04-wrolpi/03-run-chroot.sh
View file

@ -18,7 +18,7 @@ useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
sed -i 's/port =.*/port = 5432/' /etc/postgresql/15/main/postgresql.conf
# Install Node console commands.
npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0
npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
# Put the latest WROLPi master in /opt/wrolpi.
git clone -b master https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi

2
scripts/install_debian_12.sh
View file

@ -14,7 +14,7 @@ node -v
npm -v
# Install serve, and archiving tools.
single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0
single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
# Build React app in background job.
cd /opt/wrolpi/app || exit 5

2
scripts/install_raspberrypios.sh
View file

@ -14,7 +14,7 @@ node -v
npm -v
# Install serve, and archiving tools.
single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0
single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
# Build React app in background job.
cd /opt/wrolpi/app || exit 5

12
wrolpi/admin.py
View file

@ -2,7 +2,7 @@ import asyncio
import enum
import subprocess
from wrolpi.cmd import NMCLI_BIN, CPUFREQ_INFO_BIN, SUDO_BIN, CPUFREQ_SET_BIN
from wrolpi.cmd import NMCLI_BIN, CPUFREQ_INFO_BIN, SUDO_BIN, CPUFREQ_SET_BIN, run_command
from wrolpi.common import logger, WROLPI_CONFIG, get_warn_once, background_task
from wrolpi.errors import ShutdownFailed
from wrolpi.events import Events
@ -203,10 +203,8 @@ async def shutdown(reboot: bool = False, delay: int = 10):
try:
# Call shutdown (or reboot) with no delay now that we have slept.
cmd = f'{SUDO_BIN} shutdown {reboot} 0'
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
cmd = [SUDO_BIN, 'shutdown', 'reboot', '0']
result = await run_command(cmd)
except subprocess.CalledProcessError as e:
if reboot:
Events.send_shutdown_failed('Reboot failed')
@ -214,12 +212,12 @@ async def shutdown(reboot: bool = False, delay: int = 10):
Events.send_shutdown_failed('Shutdown failed')
raise ShutdownFailed() from e
if proc.returncode != 0:
if result.return_code != 0:
if reboot:
Events.send_shutdown_failed('Reboot failed')
else:
Events.send_shutdown_failed('Shutdown failed')
raise ShutdownFailed(f'Got return code {proc.returncode}')
raise ShutdownFailed(f'Got return code {result.return_code}')
background_task(_())
if reboot:

4
wrolpi/captions.py
View file

@ -60,7 +60,7 @@ def read_captions(caption_path: Path):
logger.debug(f'Failed to parse caption file {caption_path}')
def extract_captions(path: pathlib.Path) -> Optional[str]:
def extract_captions(path: pathlib.Path) -> str | None:
"""Extract captions that are embedded in a video file."""
with tempfile.TemporaryDirectory() as directory:
directory = pathlib.Path(directory)
@ -80,4 +80,4 @@ def extract_captions(path: pathlib.Path) -> Optional[str]:
return captions
# No captions could be extracted.
return
return None

91
wrolpi/cmd.py
View file

@ -1,10 +1,16 @@
"""Paths for third-party commands that WROLPi may require."""
import asyncio
import dataclasses
import os
import pathlib
import shutil
import tempfile
from asyncio import CancelledError
from pathlib import Path
from time import time
from typing import Optional
from wrolpi.common import logger
from wrolpi.common import logger, TRACE_LEVEL
from wrolpi.vars import PYTEST, DOCKERIZED
logger = logger.getChild(__name__)
@ -86,3 +92,86 @@ YT_DLP_BIN = which(
FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe')
FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg')
JC_BIN = which('jc', '/usr/bin/jc')
@dataclasses.dataclass
class CommandResult:
return_code: int
cancelled: bool
stdout: bytes
stderr: bytes
elapsed: int
pid: int = None
TESTING_RUN_COMMAND_RESULT = None
async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
timeout: int = 600, debug: bool = False) -> CommandResult:
"""Run a shell command, return the results (stdout/stderr/return code).
:param debug: Enable debug logging (disabled by default because this is used by very frequent commands).
"""
if not isinstance(cmd, (list, tuple)):
raise RuntimeError('Command must be a list or tuple')
if PYTEST and TESTING_RUN_COMMAND_RESULT:
logger.debug('run_command: returning mock result')
return TESTING_RUN_COMMAND_RESULT() # call the mock
with tempfile.NamedTemporaryFile() as stdout_fh, tempfile.NamedTemporaryFile() as stderr_fh:
stdout_file, stderr_file = pathlib.Path(stdout_fh.name), pathlib.Path(stderr_fh.name)
cmd = tuple(str(i) for i in cmd)
if debug:
logger.debug(f'run_command: running ({timeout=}): {cmd=}')
elif logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'run_command: running ({timeout=}): {cmd=}')
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=stdout_fh, # Use stdout/stderr files to avoid buffer filling up.
stderr=stderr_fh,
cwd=cwd,
)
pid = proc.pid
cancelled = False
start = time()
try:
while True:
elapsed = int(time() - start)
if timeout and elapsed >= timeout:
logger.warning(f'run_command: timeout exceeded, killing... {cmd=}')
proc.kill()
await proc.wait()
break
try:
# Wait for the process to finish.
await asyncio.wait_for(proc.wait(), timeout=1)
break
except asyncio.TimeoutError:
# Task is not done, keep waiting...
continue
except CancelledError as e:
logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e)
cancelled = True
proc.kill()
await proc.wait()
elapsed = int(time() - start)
stdout = stdout_file.read_bytes() or b''
stderr = stderr_file.read_bytes() or b''
# Logs details of the call, but only if it took a long time or TRACE is enabled.
if debug:
logger.debug(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}')
elif logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}')
return CommandResult(
return_code=proc.returncode,
cancelled=cancelled,
stdout=stdout,
stderr=stderr,
elapsed=elapsed,
pid=pid,
)

11
wrolpi/common.py
View file

@ -1732,6 +1732,15 @@ async def cancel_background_tasks():
await asyncio.gather(*BACKGROUND_TASKS)
async def await_background_tasks():
"""Awaits all background tasks, used only for testing."""
if not PYTEST:
raise RuntimeError('await_background_tasks should only be used for testing')
for background_task in BACKGROUND_TASKS.copy():
await background_task
def get_warn_once(message: str, logger__: logging.Logger, level=logging.ERROR):
"""Create a function that will report an error only once.
@ -1985,7 +1994,7 @@ def format_json_file(file: pathlib.Path, indent: int = 2):
replace_file(file, content)
def format_html_string(html: str) -> str:
def format_html_string(html: str | bytes) -> str:
if not html.strip():
raise RuntimeError('Refusing to format empty HTMl string.')

36
wrolpi/conftest.py
View file

@ -34,8 +34,9 @@ from sqlalchemy.orm import Session, sessionmaker
import wrolpi.root_api # noqa
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import CommandResult
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL
from wrolpi.common import logger
from wrolpi.common import logger, await_background_tasks as await_background_tasks_
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
from wrolpi.dates import set_test_now
@ -202,6 +203,13 @@ async def await_switches(async_client):
return await_switches_
@pytest.fixture
async def await_background_tasks(async_client):
"""Returns the `switches.await_switches` function, but primes the Sanic App to actually handle signals."""
await async_client.get('/api')
return await_background_tasks_
@pytest.fixture
async def test_download_manager(
async_client,
@ -496,6 +504,32 @@ def mock_create_subprocess_shell() -> Callable:
return mocker
@pytest.fixture
def mock_run_command():
"""Mock wrolpi.cmd's `run_command` using `TESTING_RUN_COMMAND_RESULT`."""
default = CommandResult(
return_code=0,
cancelled=False,
stdout=b'mock_run_command fixture stdout',
stderr=b'mock_run_command fixture stderr',
elapsed=0,
pid=123,
)
with mock.patch('wrolpi.cmd.TESTING_RUN_COMMAND_RESULT') as mock_TESTING_RUN_COMMAND_RESULT:
mock_TESTING_RUN_COMMAND_RESULT.return_value = default
def do_mock(new_result):
mock_TESTING_RUN_COMMAND_RESULT.return_value = new_result
yield do_mock
@pytest.fixture
def run_command_bad_result() -> CommandResult:
return CommandResult(1, False, b'run_command_bad_result stdout', b'run_command_bad_result stderr', 0, 123)
@pytest.fixture
def events_history(async_client):
"""Give each test its own Events history."""

83
wrolpi/downloader.py
View file

@ -1,12 +1,10 @@
import asyncio
import contextlib
import hashlib
import inspect
import logging
import multiprocessing
import os
import pathlib
import re
import tempfile
import traceback
import xml.etree.ElementTree as ET
@ -29,10 +27,10 @@ from sqlalchemy.sql import Delete
from wrolpi import flags
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_wrolpi_config, get_absolute_media_path, timer, aiohttp_get, \
get_download_info, trim_file_name
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
get_download_info, trim_file_name, get_wrolpi_config
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError
@ -300,61 +298,19 @@ class Downloader:
def already_downloaded(self, *urls: List[str], session: Session = None):
raise NotImplementedError()
async def process_runner(self, download_id: int, url: str, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
timeout: int = None, **kwargs) -> Tuple[int, dict, bytes]:
async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
timeout: int = None) -> CommandResult:
"""
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
Global timeout takes precedence over the timeout argument, unless it is 0. (Smaller global timeout wins)
"""
str_cmd = " ".join([str(i) for i in cmd])
logger.info(f'{self} launching download process with args: {str_cmd}')
start = now()
# Use temporary files to prevent truncating of stdout/stderr.
with tempfile.NamedTemporaryFile() as stdout_file, tempfile.NamedTemporaryFile() as stderr_file:
stdout_file, stderr_file = pathlib.Path(stdout_file.name), pathlib.Path(stderr_file.name)
with stdout_file.open('wb') as stdout_fh, stderr_file.open('wb') as stderr_fh:
# stdout/stderr is written to real files.
proc = await asyncio.create_subprocess_exec(*cmd,
stdout=stdout_fh,
stderr=stderr_fh,
cwd=cwd,
**kwargs)
pid = proc.pid
if cwd and not cwd.is_dir():
raise RuntimeError('cwd directory does not exist')
# Timeout can be any positive integer. Global download timeout takes precedence, unless it is 0.
# A timeout of 0 means the download will never be killed.
timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
logger.debug(f'{self} launched download process {pid=} {timeout=} for {url}')
while True:
# Calculate the seconds since the process started.
elapsed = int((now() - start).total_seconds())
# Check if the cancel event is set or if the timeout is reached
if download_manager.download_is_killed(download_id):
logger.warning(f'Killing download {pid=}, {elapsed} seconds elapsed.')
proc.kill()
await proc.wait()
break
if timeout and elapsed > timeout:
# TODO this logic should be handled by the Download Manager.
download_manager.kill_download(download_id)
continue
try:
# Wait for the process to finish.
await asyncio.wait_for(proc.wait(), timeout=1)
break
except TimeoutError:
# Subprocess has not yet finished, loop again and wait for it to finish.
continue
# Read stdout/stderr files.
stdout, stderr = stdout_file.read_bytes(), stderr_file.read_bytes()
stdout_len, stderr_len = len(stdout) if stdout else 0, len(stderr) if stderr else 0
logger.debug(f'Download exited with {proc.returncode} and stdout={stdout_len} stderr={stderr_len}')
return proc.returncode, dict(stdout=stdout, stderr=stderr), stdout
timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
coro = run_command(cmd, cwd=cwd, timeout=timeout)
return await self.cancel_wrapper(coro, download)
@staticmethod
async def cancel_wrapper(coro: Coroutine, download: Download):
@ -404,8 +360,8 @@ class Downloader:
return None
async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True,
check_for_md5: bool = True) -> pathlib.Path:
async def download_file(self, download: Download, url: str, destination: pathlib.Path, check_for_meta4: bool = True) \
-> pathlib.Path:
from wrolpi.files.lib import glob_shared_stem
if not ARIA2C_PATH:
@ -439,18 +395,11 @@ class Downloader:
'-M', meta4_path,
)
return_code, logs, stdout = await self.process_runner(
download_id,
url, # Log the original URL.
cmd,
destination,
)
result = await self.process_runner(download, cmd, destination)
error = result.stderr.decode()
error = logs.get('stderr') or logs.get('stdout')
error = error.decode() if error else error
if return_code != 0:
raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}')
if result.return_code != 0:
raise DownloadError(f'{error}\n\nFileDownloader failed with return code {result.return_code}')
if not output_path.is_file():
raise DownloadError(f'{error}\n\nOutput file not found: {output_path}')

4
wrolpi/events.py
View file

@ -92,6 +92,10 @@ class Events:
def send_config_save_failed(message: str = None):
send_event('config_save_failed', message, subject='configs')
@classmethod
def send_archive_uploaded(cls, message: str = None, url: str = None):
send_event('upload_archive', message, subject='upload', url=url)
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
log = f'{event=}'

5
wrolpi/files/api.py
View file

@ -45,6 +45,9 @@ async def get_file(_: Request, body: schema.FileRequest):
file = lib.get_file_dict(body.file)
except FileNotFoundError:
raise InvalidFile()
# TODO ensure that this is being called during file previews and preview on page load
background_task(lib.set_file_viewed(get_media_directory() / body.file))
return json_response({'file': file})
@ -102,7 +105,7 @@ async def post_search_files(_: Request, body: schema.FilesSearchRequest):
with timer('Searching all files', 'info', logger__=logger):
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
body.tag_names, body.headline, body.months, body.from_year, body.to_year,
body.any_tag)
body.any_tag, body.order)
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))

3
wrolpi/files/downloader.py
View file

@ -1,5 +1,4 @@
import pathlib
import pathlib
import traceback
from abc import ABC
@ -52,7 +51,7 @@ class FileDownloader(Downloader, ABC):
output_path = None
try:
output_path = await self.download_file(download.id, download.url, destination)
output_path = await self.download_file(download, download.url, destination)
fg = await upsert_file(output_path, tag_names=download.tag_names)
location = fg.location
return DownloadResult(

24
wrolpi/files/lib.py
View file

@ -90,6 +90,18 @@ def get_file_dict(file: str) -> Dict:
return _get_file_dict(media_directory / file)
@optional_session
async def set_file_viewed(file: pathlib.Path, session: Session = None):
"""Change FileGroup.viewed to the current datetime."""
try:
fg = FileGroup.find_by_path(file, session)
except UnknownFile:
fg = FileGroup.from_paths(session, file)
fg.do_model(session)
fg.set_viewed()
session.commit()
@cachetools.func.ttl_cache(10_000, 30.0)
def _get_directory_dict(directory: pathlib.Path,
directories_cache: str, # Used to cache by requested directories.
@ -774,7 +786,7 @@ async def search_directories_by_name(name: str, excluded: List[str] = None, limi
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
from_year: int = None, to_year: int = None, any_tag: bool = False) -> \
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None) -> \
Tuple[List[dict], int]:
"""Search the FileGroup table.
@ -792,6 +804,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
@param tag_names: A list of tag names.
@param headline: Includes Postgresql headline if True.
@param months: A list of integers representing the index of the month of the year, starting at 1.
@param order: Used to change results from most relevant to recently viewed.
"""
params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%')
wheres = []
@ -823,6 +836,15 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
else:
headline = ''
if order == 'viewed':
order_by = 'viewed DESC NULLS LAST, 1 ASC'
elif order == '-viewed':
order_by = 'viewed ASC NULLS LAST, 1 ASC'
if order and not search_str:
# Only filter out unviewed files if search_str is not provided.
wheres.append('viewed IS NOT NULL')
wheres = '\n AND '.join(wheres)
selects = f"{', '.join(selects)}, " if selects else ""
join = '\n'.join(joins)

24
wrolpi/files/models.py
View file

@ -1,7 +1,6 @@
import copy
import pathlib
import shutil
import urllib.parse
from datetime import datetime
from functools import singledispatchmethod
from typing import List, Type, Optional, Iterable
@ -11,7 +10,7 @@ from sqlalchemy import types
from sqlalchemy.orm import deferred, relationship, Session
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
get_relative_to_media_directory, unique_by_predicate
unique_by_predicate, TRACE_LEVEL
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
from wrolpi.db import optional_session
from wrolpi.downloader import Download
@ -141,8 +140,13 @@ class FileGroup(ModelHelper, Base):
return self.title
return self.primary_path.name
def set_viewed(self):
self.viewed = now()
def set_viewed(self, viewed: datetime = None) -> datetime:
"""
:param viewed: Used only for testing!
:return: The datetime that was set.
"""
self.viewed = viewed or now()
return self.viewed
@optional_session
def set_tags(self, tag_names_or_ids: Iterable[str | int], session: Session = None):
@ -308,7 +312,6 @@ class FileGroup(ModelHelper, Base):
raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem: {paths[0]}')
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
logger.debug(f'FileGroup.from_paths: {len(existing_groups)}')
if len(existing_groups) == 0:
# These paths have not been used previously, create a new FileGroup.
file_group = FileGroup()
@ -336,7 +339,8 @@ class FileGroup(ModelHelper, Base):
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
file_group.size = sum(i.stat().st_size for i in paths)
file_group.mimetype = get_mimetype(file_group.primary_path)
logger.debug(f'FileGroup.from_paths: {file_group}')
if logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'FileGroup.from_paths: {file_group}')
return file_group
@ -353,6 +357,14 @@ class FileGroup(ModelHelper, Base):
file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
return file_group
@staticmethod
@optional_session
def find_by_path(path, session) -> 'FileGroup':
file_group = FileGroup.get_by_path(path, session)
if not file_group:
raise UnknownFile(f'Unable to find FileGroup with path {path}')
return file_group
@property
def tag_names(self) -> List[str]:
return [i.tag.name for i in self.tag_files]

1
wrolpi/files/schema.py
View file

@ -32,6 +32,7 @@ class FilesSearchRequest:
from_year: Optional[int] = None
to_year: Optional[int] = None
any_tag: bool = False
order: Optional[str] = None
def __post_init__(self):
if self.any_tag and self.tag_names:

46
wrolpi/files/test/test_api.py
View file

@ -1,5 +1,6 @@
import hashlib
import json
from datetime import datetime, timezone
from http import HTTPStatus
from unittest import mock
@ -160,6 +161,42 @@ async def test_delete_wrol_mode(async_client, wrol_mode_fixture):
assert response.json['code'] == 'WROL_MODE_ENABLED'
@pytest.mark.asyncio
async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory):
"""File search can return the most recently viewed files."""
video_file_factory(test_directory / 'foo.mp4'), video_file_factory(test_directory / 'bar.mp4')
baz = (test_directory / 'baz.txt')
baz.write_text('baz contents')
await lib.refresh_files()
# baz is most recently viewed, foo has not been viewed.
bar, baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
bar.set_viewed(datetime(2000, 1, 1, 1, 1, 1, tzinfo=timezone.utc))
baz.set_viewed(datetime(2000, 1, 1, 1, 1, 2, tzinfo=timezone.utc))
test_session.commit()
# Results match the viewed order. foo is not viewed.
body = dict(order='viewed')
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], response.json
assert [i['name'] for i in response.json['file_groups']] == ['baz.txt', 'bar.mp4']
# Results match the reverse viewed order. foo is not viewed.
body = dict(order='-viewed')
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], response.json
assert [i['name'] for i in response.json['file_groups']] == ['bar.mp4', 'baz.txt']
# foo is included because `search_str` has a value.
body = dict(search_str='foo', order='viewed')
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], response.json
assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4']
def test_files_search(test_session, test_client, make_files_structure, assert_files_search):
# You can search an empty directory.
assert_files_search('nothing', [])
@ -757,7 +794,7 @@ async def test_delete_directory_recursive(test_session, test_directory, make_fil
@pytest.mark.asyncio
async def test_get_file(test_session, async_client, test_directory, make_files_structure):
async def test_get_file(test_session, async_client, test_directory, make_files_structure, await_background_tasks):
"""Can get info about a single file."""
make_files_structure({'foo/bar.txt': 'foo contents'})
await lib.refresh_files()
@ -768,6 +805,13 @@ async def test_get_file(test_session, async_client, test_directory, make_files_s
assert response.json['file']
assert response.json['file']['path'] == 'foo/bar.txt'
# Wait for `viewed` to be set in background task.
await await_background_tasks()
fg, = test_session.query(FileGroup).all()
assert fg.primary_path == test_directory / 'foo/bar.txt'
assert isinstance(fg.viewed, datetime)
@pytest.mark.asyncio
async def test_ignore_directory(test_session, async_client, test_directory, make_files_structure, test_wrolpi_config,

3
wrolpi/files/test/test_scrape_downloader.py
View file

@ -30,7 +30,8 @@ async def fake_file_do_download(*a, **kwargs):
return DownloadResult(success=True)
async def test_scrape_html_downloader(test_directory, test_session, test_download_manager, assert_download_urls, await_switches):
async def test_scrape_html_downloader(test_directory, test_session, test_download_manager, assert_download_urls,
await_switches):
"""User can scrape for PDF files."""
test_download_manager.register_downloader(ScrapeHTMLDownloader())
test_download_manager.register_downloader(FileDownloader())

1
wrolpi/schema.py
View file

@ -370,4 +370,3 @@ class SearchFileEstimateRequest:
@dataclass
class SearchOtherEstimateRequest:
tag_names: List[str] = field(default_factory=list)

42
wrolpi/status.py
View file

@ -14,7 +14,7 @@ from typing import List, Optional, Dict
import jc
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which
from wrolpi.cmd import which, run_command
from wrolpi.common import logger, get_warn_once, unique_by_predicate, partition, TRACE_LEVEL
from wrolpi.dates import now
@ -86,14 +86,11 @@ class IostatInfo:
async def get_iostat_stats() -> IostatInfo:
try:
proc = await asyncio.create_subprocess_shell(
'iostat 3 2', # 3 second interval to gather more data, get 2 count.
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
result = await run_command(
('iostat', '3', '2'), # 3 second interval to gather more data, get 2 count.
)
stdout, stderr = await proc.communicate()
if stdout and proc.returncode == 0:
iostat_stats = jc.parse('iostat', stdout.decode(), quiet=True)
if result.stdout and result.return_code == 0:
iostat_stats = jc.parse('iostat', result.stdout.decode(), quiet=True)
if iostat_stats:
cpu_statuses, _ = partition(lambda i: i['type'] == 'cpu', iostat_stats)
_, cpu_stats = cpu_statuses # Use the second status because the first is a lie.
@ -216,21 +213,16 @@ async def get_cpu_stats() -> CPUInfo:
return info
PS_CMD = 'ps aux --sort=-%cpu --cols=512'
IGNORED_PROCESS_COMMANDS = {PS_CMD, '<defunct>'}
PS_CMD = ('ps', 'aux', '--sort=-%cpu', '--cols=512')
IGNORED_PROCESS_COMMANDS = {' '.join(PS_CMD), '<defunct>'}
async def get_processes_stats() -> List[ProcessInfo]:
proc = await asyncio.create_subprocess_shell(
PS_CMD,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
result = await run_command(PS_CMD)
# Do not include `ps` in the top processes.
ps_pid = proc.pid
stdout, stderr = await proc.communicate()
ps_pid = result.pid
ps_info = jc.parse('ps', stdout.decode(), quiet=True)
ps_info = jc.parse('ps', result.stdout.decode(), quiet=True)
processes = sorted(ps_info, key=lambda i: i['cpu_percent'], reverse=True)
processes = [i for i in processes if i['pid'] != ps_pid
and i['cpu_percent'] > 1
@ -553,15 +545,11 @@ async def get_power_stats() -> PowerStats:
logger.trace(f'get_power_stats called')
try:
proc = await asyncio.create_subprocess_shell(
'dmesg',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if stdout and proc.returncode == 0:
under_voltage = b' Undervoltage detected' in stdout
over_current = b' over-current change ' in stdout
cmd = ('dmesg',)
result = await run_command(cmd)
if result.stdout and result.return_code == 0:
under_voltage = b' Undervoltage detected' in result.stdout
over_current = b' over-current change ' in result.stdout
return PowerStats(
under_voltage=under_voltage,
over_current=over_current,

44
wrolpi/test/test_cmd.py
View file

@ -1,3 +1,4 @@
import asyncio
import os
from pathlib import Path
@ -20,3 +21,46 @@ def test_which(test_directory):
assert cmd.which('asdf', str(foo)) is None
os.chmod(foo, 0o777)
assert cmd.which('asdf', str(foo)) == foo
TEST_SCRIPT = '''#!/bin/sh
echo stdout
echo stderr >&2
'''
async def test_run_command(test_directory):
"""Can read both stdin and stderr. Can get return code."""
(test_directory / 'script.sh').write_text(TEST_SCRIPT)
result = await cmd.run_command(('sh', 'script.sh'), cwd=test_directory)
assert result.return_code == 0
assert result.cancelled is False
assert result.stdout == b'stdout\n'
assert result.stderr == b'stderr\n'
assert result.elapsed == 0
async def test_run_command_timeout(test_directory):
"""Timeout is obeyed."""
result = await cmd.run_command(('sleep', '10'), timeout=2)
assert result.return_code == -9
assert result.cancelled is False
assert result.stdout == b''
assert result.stderr == b''
assert result.elapsed >= 2
async def test_run_command_cancel(test_directory):
"""Run a command, cancel it and test that it was cancelled."""
task = asyncio.create_task(cmd.run_command(('sleep', '10')))
try:
await asyncio.wait_for(task, timeout=1)
except TimeoutError:
task.cancel()
result = await task
assert result.return_code == -9
assert result.cancelled is True
assert result.stdout == b''
assert result.stderr == b''
assert result.elapsed == 1

61
wrolpi/test/test_downloader.py
View file

@ -1,7 +1,6 @@
import asyncio
import json
import pathlib
import tempfile
from abc import ABC
from datetime import datetime
from http import HTTPStatus
@ -303,26 +302,26 @@ async def test_process_runner_timeout(async_client, test_session, test_directory
# Sleep for 8 seconds really takes 8 seconds.
start = datetime.now()
await downloader.process_runner(1, 'https://example.com', ('sleep', '8'), test_directory, timeout=10)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=10)
elapsed = datetime.now() - start
assert elapsed.total_seconds() >= 8
# Default timeout is obeyed.
# Download.timeout is obeyed.
start = datetime.now()
await downloader.process_runner(1, 'https://example.com', ('sleep', '8'), test_directory)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory)
elapsed = datetime.now() - start
assert 3 < elapsed.total_seconds() < 5
# One-off timeout is obeyed.
start = datetime.now()
await downloader.process_runner(2, 'https://example.com', ('sleep', '8'), test_directory, timeout=1)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=1)
elapsed = datetime.now() - start
assert 1 < elapsed.total_seconds() < 3
# Global timeout is obeyed.
get_wrolpi_config().download_timeout = 3
start = datetime.now()
await downloader.process_runner(3, 'https://example.com', ('sleep', '8'), test_directory)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory)
elapsed = datetime.now() - start
assert 2 < elapsed.total_seconds() < 5
@ -341,34 +340,30 @@ exit 123
@pytest.mark.asyncio
async def test_process_runner_output(async_client, test_directory, test_downloader):
with tempfile.TemporaryDirectory() as directory:
directory = pathlib.Path(directory)
script = directory / 'echo_script.sh'
script.write_text(GOOD_SCRIPT)
cmd = ('/bin/bash', script)
return_code, logs, stdout = await test_downloader.process_runner(
1,
'https://example.com',
cmd,
directory,
timeout=1
)
assert return_code == 0
assert logs['stdout'] == b'standard output\n'
assert logs['stderr'] == b'standard error\n'
script = test_directory / 'echo_script.sh'
script.write_text(GOOD_SCRIPT)
cmd = ('/bin/bash', script)
result = await test_downloader.process_runner(
Download(),
cmd,
test_directory,
timeout=1
)
assert result.return_code == 0
assert result.stdout == b'standard output\n'
assert result.stderr == b'standard error\n'
script.write_text(BAD_SCRIPT)
cmd = ('/bin/bash', script)
return_code, logs, stdout = await test_downloader.process_runner(
1,
'https://example.com',
cmd,
directory,
timeout=1
)
assert return_code == 123
assert logs['stdout'] == b'bad standard output\n'
assert logs['stderr'] == b'bad standard error\n'
script.write_text(BAD_SCRIPT)
cmd = ('/bin/bash', script)
result = await test_downloader.process_runner(
Download(),
cmd,
test_directory,
timeout=1
)
assert result.return_code == 123
assert result.stdout == b'bad standard output\n'
assert result.stderr == b'bad standard error\n'
@pytest.mark.asyncio

2
wrolpi/test/test_rss.py
View file

@ -104,7 +104,7 @@ async def test_rss_downloader_filter_titles(test_session):
rss_downloader = RSSDownloader()
entries = [
dict(link='https://example.com/a', title='A'), # Case should be ignored.
dict(link='https://example.com/a', title='A'), # Case should be ignored.
dict(link='https://example.com/b', title='b'),
dict(link='https://example.com/c', title='c'),
]