diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js index f82c20cd..d2f4ed64 100644 --- a/app/src/DashboardPage.js +++ b/app/src/DashboardPage.js @@ -1,4 +1,11 @@ -import {ErrorMessage, LoadStatistic, PageContainer, SearchResultsInput, useTitle} from "./components/Common"; +import { + ErrorMessage, + LoadStatistic, + PageContainer, + RefreshHeader, + SearchResultsInput, + useTitle +} from "./components/Common"; import React, {useContext, useState} from "react"; import {Media, SettingsContext, StatusContext} from "./contexts/contexts"; import {DownloadMenu} from "./components/Download"; @@ -25,8 +32,8 @@ import {TagsDashboard} from "./Tags"; import {Upload} from "./components/Upload"; import {SearchView, useSearch, useSearchSuggestions} from "./components/Search"; import {KiwixRestartMessage, OutdatedZimsMessage} from "./components/Zim"; -import {useSearchFilter, useWROLMode} from "./hooks/customHooks"; -import {FileSearchFilterButton} from "./components/Files"; +import {useSearchFilter, useSearchRecentFiles, useWROLMode} from "./hooks/customHooks"; +import {FileCards, FileSearchFilterButton} from "./components/Files"; import {DateSelectorButton} from "./components/DatesSelector"; import {useCalculators} from "./components/Calculators"; import {evaluate} from "mathjs"; @@ -234,6 +241,72 @@ export function Getters() { } + +function DashboardStatus() { + const {status} = useContext(StatusContext); + + let percent = 0; + let load = {}; + let cores = 0; + let pending_downloads = '?'; + if (status && status['cpu_stats']) { + percent = status['cpu_stats']['percent']; + load = status['load_stats']; + cores = status['cpu_stats']['cores']; + } + + const {downloads} = status; + if (!_.isEmpty(downloads)) { + pending_downloads = downloads && downloads['disabled'] ? 'x' : downloads['pending']; + } + + let bandwidths = ; + if (status && status['nic_bandwidth_stats']) { + bandwidths = Object.entries(status['nic_bandwidth_stats']) + .map(([name, bandwidth]) => ); + } else if (window.apiDown) { // apiDown is set in useStatus + bandwidths = null; + } + + return + +
Status
+ + + + + + + + +
Bandwidth
+ {bandwidths} + + + + + + + + + + +
; +} + +function DashboardRecentFiles() { + const {searchFiles, loading, fetchFiles} = useSearchRecentFiles(); + + return + + + +} + export function DashboardPage() { const navigate = useNavigate(); @@ -284,6 +357,7 @@ export function DashboardPage() { + ; if (searchStr || (activeTags && activeTags.length > 0)) { // User has submitted and wants full search. @@ -370,56 +444,4 @@ export function DashboardPage() { {!searchStr && } {body} -} - -function DashboardStatus() { - const {status} = useContext(StatusContext); - - let percent = 0; - let load = {}; - let cores = 0; - let pending_downloads = '?'; - if (status && status['cpu_stats']) { - percent = status['cpu_stats']['percent']; - load = status['load_stats']; - cores = status['cpu_stats']['cores']; - } - - const {downloads} = status; - if (!_.isEmpty(downloads)) { - pending_downloads = downloads && downloads['disabled'] ? 'x' : downloads['pending']; - } - - let bandwidths = ; - if (status && status['nic_bandwidth_stats']) { - bandwidths = Object.entries(status['nic_bandwidth_stats']) - .map(([name, bandwidth]) => ); - } else if (window.apiDown) { // apiDown is set in useStatus - bandwidths = null; - } - - return - -
Status
- - - - - - - - -
Bandwidth
- {bandwidths} - - - - - - - - - - -
; -} +} \ No newline at end of file diff --git a/app/src/Events.js b/app/src/Events.js index 01dd297d..29a35412 100644 --- a/app/src/Events.js +++ b/app/src/Events.js @@ -92,6 +92,15 @@ function handleEvents(events) { eventToast('Config Save Failed', message, 'error'); } + if (event === 'upload_archive') { + eventToast( + 'Archive Uploaded', + message, + 'success', + 5000, + () => window.open(url, '_self')); + } + if (subject) { newestEvents[subject] = dt; } diff --git a/app/src/api.js b/app/src/api.js index 6cb47dd1..8c833a39 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -222,6 +222,22 @@ export async function getVideo(video_id) { return [data['file_group'], data['prev'], data['next']]; } +export async function getVideoComments(videoId) { + const response = await apiGet(`${VIDEOS_API}/video/${videoId}/comments`); + let data = await response.json(); + return { + comments: data.comments, + } +} + +export async function getVideoCaptions(videoId) { + const response = await apiGet(`${VIDEOS_API}/video/${videoId}/captions`); + let data = await response.json(); + return { + captions: data.captions, + } +} + export async function deleteVideos(videoIds) { if (!videoIds || videoIds.length === 0) { toast({ @@ -827,7 +843,7 @@ export async function deleteDownload(downloadId) { } } -export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag) { +export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order) { const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag}; if (mimetypes) { body['mimetypes'] = mimetypes; @@ -850,6 +866,9 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta if (toYear) { body['to_year'] = toYear; } + if (order) { + body['order'] = order; + } console.info('searching files', body); const response = await apiPost(`${API_URI}/files/search`, body); @@ -1171,20 +1190,6 @@ export async function deleteTag(id, name) { } } -export async function fetchFile(path) { - const uri = `${API_URI}/files/file`; - const body = {file: path}; - const response = await apiPost(uri, body); - if (response.ok) { - const content = await response.json(); - return content['file']; - } else { - console.error('Unable to fetch file dict! See client logs.'); - const message = await getErrorMessage(response, 'Unable to get File'); - toast({type: 'error', title: 'File Error', description: message, time: 5000}); - } -} - export async function sendNotification(message, url) { const body = {message, url}; const response = await apiPost(`${API_URI}/notify`, body); diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index b11ed6fd..e6c04efe 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -5,7 +5,6 @@ import { CardHeader, CardMeta, Container, - Dropdown, GridColumn, GridRow, Image, @@ -24,8 +23,8 @@ import { ExternalCardLink, FileIcon, findPosterPath, - InfoHeader, humanFileSize, + InfoHeader, isoDatetimeToAgoPopup, mimetypeColor, PageContainer, @@ -39,7 +38,7 @@ import { import {deleteArchives, postDownload, tagFileGroup, untagFileGroup} from "../api"; import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; -import {useArchive, useDomains, useSearchArchives, useSearchDomain, useSearchOrder} from "../hooks/customHooks"; +import {useArchive, useDomains, useSearchArchives, useSearchOrder} from "../hooks/customHooks"; import {FileCards, FileRowTagIcon, FilesView} from "./Files"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import _ from "lodash"; @@ -48,7 +47,7 @@ import {Button, Card, CardIcon, darkTheme, Header, Loader, Placeholder, Popup, S import {SortableTable} from "./SortableTable"; import {taggedImageLabel, TagsSelector} from "../Tags"; import {toast} from "react-semantic-toasts-2"; -import {Downloaders} from "./Vars"; +import {API_ARCHIVE_UPLOAD_URI, Downloaders} from "./Vars"; function archiveFileLink(path, directory = false) { if (path) { @@ -394,29 +393,47 @@ export function DomainsPage() { ; } -export function SearchDomain() { - // A Dropdown which allows the user to filter by Archive domains. - const {domain, domains, setDomain} = useSearchDomain(); +function ArchiveSettingsPage() { + const {t} = React.useContext(ThemeContext); - const handleChange = (e, {value}) => { - setDomain(value); - } + const urlClipboardButton = navigator.clipboard.writeText(API_ARCHIVE_UPLOAD_URI)} + />; + const dataFieldNameClipboardButton = navigator.clipboard.writeText('singlefile_contents')} + />; + const urlFieldNameClipboardButton = navigator.clipboard.writeText('url')} + />; - let domainOptions = []; + return +
SingleFile Browser Extension
- if (domains && domains.length > 0) { - domainOptions = domains.map(i => { - return {key: i['domain'], value: i['domain'], text: i['domain']} - }); - } - return <> - + These are the settings necessary to configure the SingleFile Browser + Extension to automatically upload to your WROLPi. +

+ + + - + + + + +
} function ArchivesPage() { @@ -599,12 +616,14 @@ export function ArchiveRoute() { const links = [ {text: 'Archives', to: '/archive', end: true}, {text: 'Domains', to: '/archive/domains'}, + {text: 'Settings', to: '/archive/settings'}, ]; return }/> }/> + }/> }/> diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 8fff23ca..5e10e8e3 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -10,6 +10,8 @@ import { Container, Dimmer, DimmerDimmable, + GridColumn, + GridRow, Icon as SIcon, IconGroup, Input, @@ -48,6 +50,7 @@ import {FilePreviewContext} from "./FilePreview"; import _ from "lodash"; import {killDownloads, startDownloads} from "../api"; import {allFrequencyOptions, NAME, semanticUIColorMap, validUrlRegex} from "./Vars"; +import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; export function Paginator({activePage, onPageChange, totalPages, showFirstAndLast, size = 'mini'}) { const handlePageChange = (e, {activePage}) => { @@ -1960,14 +1963,22 @@ export function getDistinctColor(hexColors) { return hslToHex(newColor.h, newColor.s, newColor.l); } -export const DismissibleMessage = ({storageName, children}) => { - const [dismissed, setDismissed] = useLocalStorage(storageName, false); - - if (dismissed) { - return +export const RefreshHeader = ({header, headerSize = 'h2', onRefresh, popupContents}) => { + const refreshButton = ; + let popup; + if (popupContents) { + popup = ; } - - return setDismissed(true)}> - {children} - + return + + +
{header}
+
+ {popup || refreshButton} +
+
} \ No newline at end of file diff --git a/app/src/components/FilePreview.js b/app/src/components/FilePreview.js index b1dcb6ea..e49d3b51 100644 --- a/app/src/components/FilePreview.js +++ b/app/src/components/FilePreview.js @@ -4,7 +4,7 @@ import {Grid, Header as SHeader, Image,} from "semantic-ui-react"; import {TagsSelector} from "../Tags"; import {Media} from "../contexts/contexts"; import {encodeMediaPath} from "./Common"; -import {fetchFile, getFile, tagFileGroup, untagFileGroup} from "../api"; +import {getFile, tagFileGroup, untagFileGroup} from "../api"; import {StlViewer} from "react-stl-viewer"; import {Button, Modal, ModalActions, ModalContent, ModalHeader} from "./Theme"; import {toast} from "react-semantic-toasts-2"; @@ -200,7 +200,7 @@ export function FilePreviewProvider({children}) { // Get the file again with its Tags. const {path, primary_path} = previewFile; try { - const file = await fetchFile(primary_path ?? path); + const file = await getFile(primary_path ?? path); setPreviewFile(file); } finally { await handleCallbacks(); @@ -310,6 +310,7 @@ export function FilePreviewProvider({children}) { const lowerPath = path.toLowerCase(); const url = getMediaPathURL(previewFile); const downloadURL = getDownloadPathURL(previewFile); + if (mimetype.startsWith('text/') && size > MAXIMUM_TEXT_SIZE) { // Large text files should be downloaded. window.open(downloadURL); @@ -336,6 +337,14 @@ export function FilePreviewProvider({children}) { // No special handler for this file type, just open it. setModalContent(getGenericPreviewModal(previewFile), url, downloadURL, path, taggable); } + + // Fetch the file so `FileGroup.viewed` is set. + try { + getFile(path); + } catch (e) { + console.error(e); + console.error('Failed to get file to set FileGroup.viewed'); + } } }, [previewFile]); diff --git a/app/src/components/Vars.js b/app/src/components/Vars.js index 414413ca..8a44c5df 100644 --- a/app/src/components/Vars.js +++ b/app/src/components/Vars.js @@ -12,6 +12,7 @@ export const FILES_MEDIA_URI = `https://${window.location.hostname}:${port}/medi export const MAP_VIEWER_URI = `https://${window.location.hostname}:8084`; export const ZIM_VIEWER_URI = `https://${window.location.hostname}:8085`; export const HELP_VIEWER_URI = `https://${window.location.hostname}:8086`; +export const API_ARCHIVE_UPLOAD_URI = `http://${window.location.hostname}:8081/api/archive/upload`; export let defaultFileOrder = '-published_datetime'; // Most recently published files first. export let defaultSearchOrder = 'rank'; // Most relevant files first. diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 234070a9..0bf08403 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -18,7 +18,7 @@ import { import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Container from "semantic-ui-react/dist/commonjs/elements/Container"; import {VideoPlaceholder} from "./Placeholder"; -import {useChannel} from "../hooks/customHooks"; +import {useChannel, useVideoCaptions, useVideoExtras} from "../hooks/customHooks"; import {ThemeContext} from "../contexts/contexts"; import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme"; import {VideoCard} from "./Videos"; @@ -192,6 +192,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { // Get the Video's channel, fallback to the URL's channel id. const {channelId} = useParams(); const {channel} = useChannel(video && video.channel_id ? video.channel_id : channelId); + const {comments, captions} = useVideoExtras(videoFile?.video?.id); if (videoFile === null) { return ; @@ -236,11 +237,11 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { const captionsPane = { menuItem: 'Captions', render: () => -
{video.caption || 'No captions available.'}
+
{captions || 'No captions available.'}
}; - const {poster_file, info_json_file, comments} = video; + const {poster_file, info_json_file} = video; const filesPane = { menuItem: 'Files', render: () =>

Video File

diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index 39d71cf1..1aa1bb63 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -12,15 +12,15 @@ import { Modal, ModalContent, ModalHeader, - Popup, Segment } from "../Theme"; import {ButtonGroup, Container, Dimmer, Dropdown, GridColumn, GridRow, Icon, Input} from "semantic-ui-react"; import { APIButton, ErrorMessage, - InfoPopup, HotspotToggle, + InfoPopup, + RefreshHeader, ThrottleToggle, Toggle, useMessageDismissal, @@ -489,16 +489,6 @@ export function SettingsPage() { body = Unable to fetch settings } - const refreshButton = ; - const refreshPopupButton = ; - const controlSegment =
Control WROLPi
@@ -509,14 +499,11 @@ export function SettingsPage() {
; const configsSegment = - - - -
Configs
-
- {refreshPopupButton} -
-
+

These configs control this WROLPi; they are the source of truth. Any changes to configs will be applied to the database when imported (typically at startup).

diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index b8890dd0..0898f771 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -18,6 +18,8 @@ import { getStatistics, getStatus, getVideo, + getVideoCaptions, + getVideoComments, getVideosStatistics, postDumpConfig, postImportConfig, @@ -423,6 +425,42 @@ export const useVideo = (videoId) => { return {videoFile, prevFile, nextFile, fetchVideo}; } +export const useVideoExtras = (videoId) => { + // Fetches extra data to display on a Video's page. + const [comments, setComments] = useState(null); + const [captions, setCaptions] = useState(null); + + const fetchComments = async () => { + try { + const result = await getVideoComments(videoId); + setComments(result.comments); + } catch (e) { + console.error(e); + } + } + + const fetchCaptions = async () => { + try { + const result = await getVideoCaptions(videoId); + setCaptions(result.captions); + } catch (e) { + console.error(e); + } + } + + useEffect(() => { + if (videoId) { + fetchComments(); + fetchCaptions(); + } else { + setComments(null); + setCaptions(null); + } + }, [videoId]); + + return {comments, captions} +} + export const useChannel = (channel_id) => { const emptyChannel = { name: '', @@ -500,6 +538,37 @@ export const useChannels = () => { return {channels, fetchChannels} } +export const useSearchRecentFiles = () => { + const [searchFiles, setSearchFiles] = useState(null); + const [loading, setLoading] = useState(false); + + const localSearchFiles = async () => { + setLoading(true); + try { + let [file_groups, total] = await filesSearch( + null, 12, null, null, null, [], false, + null, null, null, false, 'viewed'); + setSearchFiles(file_groups); + } catch (e) { + console.error(e); + toast({ + type: 'error', + title: 'Unexpected server response', + description: 'Could not get recent files', + time: 5000, + }); + } finally { + setLoading(false); + } + } + + React.useEffect(() => { + localSearchFiles(); + }, []); + + return {searchFiles, loading, fetchFiles: localSearchFiles}; +} + export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => { const { activeTags, anyTag, diff --git a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot index dad3fded..326b9325 100644 --- a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot +++ b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot @@ -20,7 +20,7 @@ node --version npm install # Install Node console commands. -npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0 +npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0 # Install Python requirements. python3 --version diff --git a/docker-compose.yml b/docker-compose.yml index f646ec4d..fd007f91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: - './alembic:/opt/wrolpi/alembic' - './alembic.ini:/opt/wrolpi/alembic.ini' ports: - - ${REACT_APP_API-127.0.0.1:8081}:8081 + - ${REACT_APP_API-0.0.0.0:8081}:8081 user: '${UID-1000}:${GID-1000}' healthcheck: test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ] diff --git a/docker/archive/Dockerfile b/docker/archive/Dockerfile index cf9789ff..c39c0c56 100644 --- a/docker/archive/Dockerfile +++ b/docker/archive/Dockerfile @@ -1,21 +1,21 @@ -# Keep older singlefile version so we can keep screenshots at specific ratio. -FROM capsulecode/singlefile:v0.3.32 +FROM browserless/chrome:1.61.1-chrome-stable USER root WORKDIR /app # Install Python for main.py ENV PYTHONUNBUFFERED=1 -RUN apk add --update --no-cache python3 python3-dev curl && ln -sf python3 /usr/bin/python -RUN python3 -m ensurepip -RUN pip3 install --no-cache --upgrade pip setuptools +RUN apt update && apt install -y python3 python3-dev python3-venv curl && ln -sf python3 /usr/bin/python +RUN python3 -m venv /app/venv +RUN . /app/venv/bin/activate + +# Install a newer singlefile and Readability. +RUN npm install -g single-file-cli@2.0.73 readability-extractor@0.0.6 # Install the Python wrapper and it's requirements. COPY docker/archive/requirements.txt /app/requirements.txt -RUN pip3 install -r /app/requirements.txt +RUN /app/venv/bin/pip3 install -r /app/requirements.txt COPY docker/archive /app -# Install a newer singlefile and Readability. -RUN npm install -g single-file-cli@1.1.54 readability-extractor@0.0.6 - -ENTRYPOINT [ "python3", "/app/main.py"] +# Launch Archive app in the virtual environment. +ENTRYPOINT [ "/app/venv/bin/python3", "/app/main.py"] diff --git a/docker/archive/main.py b/docker/archive/main.py index 9b078947..e2a98333 100644 --- a/docker/archive/main.py +++ b/docker/archive/main.py @@ -28,6 +28,8 @@ logger.addHandler(ch) SINGLEFILE_PATH = pathlib.Path('/usr/src/app/node_modules/single-file-cli/single-file') if not SINGLEFILE_PATH.is_file(): SINGLEFILE_PATH = pathlib.Path('/usr/src/app/node_modules/single-file/cli/single-file') +if not SINGLEFILE_PATH.is_file(): + SINGLEFILE_PATH = pathlib.Path('/usr/src/.nvm/versions/node/v18.19.0/bin/single-file') if not SINGLEFILE_PATH.is_file(): raise FileNotFoundError("Can't find single-file executable!") @@ -77,8 +79,6 @@ async def call_single_file(url) -> bytes: """ logger.info(f'archiving {url}') cmd = f'{SINGLEFILE_PATH}' \ - r' --browser-executable-path /usr/bin/chromium-browser' \ - r' --browser-args [\"--no-sandbox\"]' \ r' --dump-content ' \ f' "{url}"' logger.debug(f'archive cmd: {cmd}') @@ -86,7 +86,7 @@ async def call_single_file(url) -> bytes: if return_code != 0 or not stdout: if stderr: for line in stderr.splitlines(): - logger.error(line) + logger.error(line.decode()) raise RuntimeError(f'Failed to single-file {url} got the following error: {stderr}') logger.debug(f'done archiving for {url}') return stdout @@ -112,7 +112,7 @@ async def extract_readability(path: str, url: str) -> dict: async def take_screenshot(url: str) -> bytes: - cmd = '/usr/bin/chromium-browser' \ + cmd = '/usr/bin/google-chrome' \ ' --headless' \ ' --disable-gpu' \ ' --no-sandbox' \ @@ -153,8 +153,15 @@ def prepare_bytes(b: bytes) -> str: @app.post('/json') async def post_archive(request: Request): url = request.json['url'] + try: - singlefile = await call_single_file(url) + # May have been passed the singlefile contents, do the extractions and return. + singlefile = request.json.get('singlefile') + singlefile = base64.b64decode(singlefile) if singlefile else None + + # Use provided singlefile, or fetch and create from the internet. + singlefile = singlefile or await call_single_file(url) + # Use html suffix so chrome screenshot recognizes it as an HTML file. with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh: fh.write(singlefile) diff --git a/docker/archive/requirements.txt b/docker/archive/requirements.txt index db2c6767..abadf1ee 100644 --- a/docker/archive/requirements.txt +++ b/docker/archive/requirements.txt @@ -1 +1 @@ -sanic==23.3.0 +sanic==24.12.0 diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 790da176..26ada14b 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -1,18 +1,15 @@ import asyncio import json import pathlib -import tempfile from abc import ABC -from json import JSONDecodeError from typing import List, Tuple, Iterable -from selenium import webdriver from sqlalchemy import not_ from sqlalchemy.orm import Session -from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM +from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \ - slow_logger, html_screenshot, get_title_from_html + slow_logger, get_title_from_html from wrolpi.db import optional_session, get_db_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError @@ -38,6 +35,11 @@ class ArchiveDownloader(Downloader, ABC): def __repr__(self): return f'' + @optional_session + def already_downloaded(self, *urls: List[str], session: Session = None) -> List: + file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive')) + return file_groups + async def do_download(self, download: Download) -> DownloadResult: if download.attempts > 3: raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}') @@ -45,12 +47,13 @@ class ArchiveDownloader(Downloader, ABC): if DOCKERIZED or PYTEST: # Perform the archive in the Archive docker container. (Typically in the development environment). singlefile, readability, screenshot = await request_archive(download.url) + archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot) + archive_id = archive.id else: # Perform the archive using locally installed executables. - singlefile, readability, screenshot = await self.do_archive(download) - - archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot) - archive_id = archive.id + singlefile = await self.do_singlefile(download) + archive = await lib.singlefile_to_archive(singlefile) + archive_id = archive.id with get_db_session() as session: archive = Archive.find_by_id(archive_id, session) @@ -67,12 +70,7 @@ class ArchiveDownloader(Downloader, ABC): return DownloadResult(success=True, location=f'/archive/{archive.id}') - @optional_session - def already_downloaded(self, *urls: List[str], session: Session = None) -> List: - file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive')) - return file_groups - - async def do_singlefile(self, download: Download) -> Tuple[bytes, dict]: + async def do_singlefile(self, download: Download) -> bytes: """Create a Singlefile from the archive's URL.""" cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority. str(SINGLE_FILE_BIN), @@ -83,115 +81,26 @@ class ArchiveDownloader(Downloader, ABC): '--load-deferred-images-dispatch-scroll-event', download.url, ) - return_code, logs, stdout = await self.process_runner( - download.id, - download.url, - cmd, - pathlib.Path('/home/wrolpi'), - ) + cwd = pathlib.Path('/home/wrolpi') + cwd = cwd if cwd.is_dir() else None + result = await self.process_runner(download, cmd, cwd) - stderr = logs['stderr'].decode() - log_output = stderr or stdout.decode() or 'No stderr or stdout!' + stderr = result.stderr.decode() + log_output = stderr or result.stdout.decode() or 'No stderr or stdout!' - if return_code != 0: + if result.return_code != 0: e = ChildProcessError(log_output[:1000]) - raise RuntimeError(f'singlefile exited with {return_code}') from e + raise RuntimeError(f'singlefile exited with {result.return_code}') from e - if not stdout: + if not result.stdout: e = ChildProcessError(log_output[:1000]) raise RuntimeError(f'Singlefile created was empty: {download.url}') from e - if SINGLEFILE_HEADER.encode() not in stdout: + if SINGLEFILE_HEADER.encode() not in result.stdout: e = ChildProcessError(log_output[:1000]) raise RuntimeError(f'Singlefile created was invalid: {download.url}') from e - return stdout, logs - - async def do_readability(self, download: Download, html: bytes) -> dict: - """Extract the readability dict from the provided HTML.""" - with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh: - fh.write(html) - - cmd = (READABILITY_BIN, fh.name, download.url) - logger.debug(f'readability cmd: {cmd}') - return_code, logs, stdout = await self.process_runner( - download.id, - download.url, - cmd, - pathlib.Path('/home/wrolpi'), - ) - if return_code == 0: - if not stdout: - raise RuntimeError('readability stdout was empty') - try: - readability = json.loads(stdout) - except TypeError or JSONDecodeError: - # JSON was invalid. - e = ChildProcessError(stdout.decode() if stdout else 'No stdout') - if logs and (stderr := logs.get('stderr')): - e = ChildProcessError(stderr.decode()) - raise RuntimeError(f'Failed to extract readability from {download.url}') from e - logger.debug(f'done readability for {download.url}') - return readability - else: - logger.error(f'Failed to extract readability for {download.url}') - e = ChildProcessError(stdout.decode() if stdout else 'No stdout') - if logs and (stderr := logs.get('stderr')): - e = ChildProcessError(stderr.decode()) - raise RuntimeError(f'Failed to extract readability for {download.url} got {return_code}') from e - - @staticmethod - async def do_screenshot_url(download: Download) -> bytes: - """Use Chromium to get a screenshot of the Download's URL.""" - # Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of - # the page. - options = webdriver.ChromeOptions() - options.add_argument('headless') - options.add_argument('disable-gpu') - options.add_argument('window-size=1280x720') - - driver = webdriver.Chrome(chrome_options=options) - driver.get(download.url) - screenshot = driver.get_screenshot_as_png() - return screenshot - - async def do_archive(self, download: Download) -> Tuple[bytes, dict, bytes]: - """Use locally installed executables to create an Archive. - - Creates a Singlefile, readability file, and screenshot file. - - @warning: Will not raise errors if readability or screenshot cannot be extracted. - """ - singlefile, logs = await self.do_singlefile(download) - - # Extract Readability from the Singlefile. - try: - readability = await self.do_readability(download, singlefile) - except RuntimeError: - # Readability is not required. - readability = None - - screenshot = b'' - try: - # Screenshot the Singlefile first. - screenshot = html_screenshot(singlefile) - except Exception as e: - logger.error(f'Failed to screenshot file for {download.url}', exc_info=e) - - if not screenshot: - # Screenshot of singlefile failed. Download the URL again. - try: - screenshot = await self.do_screenshot_url(download) - except Exception as e: - # Screenshot failed. - logger.error(f'Failed to screenshot {download.url}', exc_info=e) - - singlefile_len = len(singlefile) if singlefile else None - readability_len = len(readability) if readability else None - screenshot_len = len(screenshot) if screenshot else None - logger.debug(f'do_archive of {download.url} finished: {singlefile_len=} {readability_len=} {screenshot_len=}') - - return singlefile, readability, screenshot + return result.stdout archive_downloader = ArchiveDownloader() @@ -258,7 +167,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive: archive.validate() archive.flush() - file_group.title = file_group.a_text = title + file_group.title = file_group.a_text = title or archive.file_group.title file_group.d_text = contents file_group.data = { 'id': archive.id, diff --git a/modules/archive/api.py b/modules/archive/api.py index dd93b21e..8dfbb4e5 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -7,7 +7,9 @@ from sanic_ext.extensions.openapi import openapi from wrolpi.api_utils import json_response from wrolpi.common import logger, wrol_mode_check, api_param_limiter from wrolpi.errors import ValidationError +from wrolpi.events import Events from wrolpi.schema import JSONErrorResponse +from wrolpi.switches import register_switch_handler from . import lib, schema NAME = 'archive' @@ -29,7 +31,7 @@ async def get_archive(_: Request, archive_id: int): @archive_bp.delete('/', name='archive_delete_one') @archive_bp.delete('/', name='archive_delete_many') -@openapi.description('Delete an individual archive') +@openapi.description('Delete Archives') @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) @wrol_mode_check async def delete_archive(_: Request, archive_ids: str): @@ -70,3 +72,38 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest): body.headline) ret = dict(file_groups=file_groups, totals=dict(file_groups=total)) return json_response(ret) + + +@register_switch_handler('singlefile_upload_switch_handler') +async def singlefile_upload_switch_handler(singlefile): + archive = await lib.singlefile_to_archive(singlefile) + logger.info(f'Created Archive from upload: {archive}') + name = archive.file_group.title or archive.file_group.url + Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location) + + +@archive_bp.get('/upload') +@openapi.definition( + summary='A message to confirm to the user that they have the correct upload URL.' +) +async def get_upload_singlefile(request: Request): + return response.text('This is the URL to upload files using the SingleFile browser extension.' + ' This requires a POST request.') + + +@archive_bp.post('/upload') +@openapi.definition( + summary='Upload SingleFile from SingleFile browser extension and convert it to an Archive.' +) +async def post_upload_singlefile(request: Request): + url = request.form['url'][0] + singlefile = request.files['singlefile_contents'][0].body + logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}') + + # Get the URL to ensure that the singlefile is valid. + lib.get_url_from_singlefile(singlefile) + + # Send processing to background task so the extension can continue. + singlefile_upload_switch_handler.activate_switch(dict(singlefile=singlefile)) + # Return empty json response because SingleFile extension expects a JSON response. + return json_response(dict(), status=HTTPStatus.OK) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 39a6672d..279981fa 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -1,10 +1,15 @@ import base64 +import base64 import gzip import json +import os import pathlib import re +import shlex +import tempfile from dataclasses import dataclass from datetime import datetime +from json import JSONDecodeError from typing import Optional, Tuple, List, Union from urllib.parse import urlparse @@ -14,13 +19,14 @@ from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive from wrolpi import dates +from wrolpi.cmd import READABILITY_BIN, run_command from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \ - format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config + format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config, html_screenshot from wrolpi.dates import now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime from wrolpi.tags import tag_append_sub_select_where -from wrolpi.vars import PYTEST +from wrolpi.vars import PYTEST, DOCKERIZED logger = logger.getChild(__name__) @@ -106,11 +112,11 @@ def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles: ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response. -async def request_archive(url: str) -> Tuple[str, Optional[str], Optional[str]]: +async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Optional[dict], Optional[str]]: """Send a request to the archive service to archive the URL.""" logger.info(f'Sending archive request to archive service: {url}') - data = {'url': url} + data = dict(url=url, singlefile=singlefile) try: async with aiohttp_post(f'{ARCHIVE_SERVICE}/json', json_=data, timeout=ARCHIVE_TIMEOUT) as response: status = response.status @@ -229,9 +235,9 @@ def get_or_create_domain(session: Session, url) -> Domain: SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n') -def get_url_from_singlefile(html: bytes) -> str: +def get_url_from_singlefile(html: bytes | str) -> str: """Extract URL from SingleFile contents.""" - html = html.decode() + html = html.decode() if isinstance(html, bytes) else html if SINGLEFILE_HEADER not in html: raise RuntimeError(f'Not a singlefile!') @@ -313,7 +319,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True except json.decoder.JSONDecodeError: # Was not valid JSON. schema = None - if isinstance(schema, dict) and schema.get('@context') in ('https://schema.org', 'http://schema.org'): + if isinstance(schema, dict) and '://schema.org' in schema.get('@context'): # Found https://schema.org/ if headline := schema.get('headline'): metadata.title = metadata.title or headline @@ -347,6 +353,17 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True metadata.author = author else: logger.warning(f'Unable to parse author schema: {author}') + elif authors := schema.get('authors'): + # Use the first Author. + author = authors[0] + + if isinstance(author, dict): + author = author.get('name') or author + + if isinstance(author, str): + metadata.author = author + else: + logger.warning(f'Unable to parse author schema: {author}') # Assume UTC if no timezone. if metadata.published_datetime and not metadata.published_datetime.tzinfo and assume_utc: @@ -368,7 +385,7 @@ def get_archive(session, archive_id: int) -> Archive: def delete_archives(*archive_ids: List[int]): - """Delete an Archive and all of it's files.""" + """Delete an Archive and all of its files.""" with get_db_session(commit=True) as session: archives: List[Archive] = list(session.query(Archive).filter(Archive.id.in_(archive_ids))) if not archives: @@ -535,3 +552,75 @@ async def search_domains_by_name(name: str, limit: int = 5, session: Session = N .limit(limit) \ .all() return domains + + +async def html_to_readability(html: str | bytes, url: str, timeout: int = 120): + """Extract the readability dict from the provided HTML.""" + with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file: + singlefile_file.write(html.encode() if isinstance(html, str) else html) + singlefile_file.flush() + os.fsync(singlefile_file.fileno()) + + cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority. + READABILITY_BIN, singlefile_file.name, shlex.quote(url)) + # Docker containers may not have this directory. But, this directory is necessary on RPi. + cwd = '/home/wrolpi' if os.path.isdir('/home/wrolpi') else None + result = await run_command(cmd, cwd=cwd, timeout=timeout, debug=True) + + logger.debug(f'readability for {url} exited with {result.return_code}') + stdout, stderr = result.stdout.decode(), result.stderr.decode() + if result.return_code == 0: + if not stdout: + raise RuntimeError('readability stdout was empty') + try: + readability = json.loads(stdout) + except TypeError or JSONDecodeError: + # JSON was invalid. + e = ChildProcessError(stdout if stdout else 'No stdout') + if stderr: + e = ChildProcessError(stderr) + raise RuntimeError(f'Failed to extract readability from {url}') from e + logger.debug(f'done readability for {url}') + return readability + else: + logger.error(f'Failed to extract readability for {url}') + e = ChildProcessError(stdout if stdout else 'No stdout') + if stderr: + e = ChildProcessError(stderr) + raise RuntimeError(f'Failed to extract readability for {url} got {result.return_code}') from e + + +async def singlefile_to_archive(singlefile: bytes) -> Archive: + """ + Convert a SingleFile to an Archive. + + This is done by extracting readability, creating a screenshot, then attaching them to an Archive/FileGroup. + """ + # Get URL first because it does some simple checking of `singlefile` + url = get_url_from_singlefile(singlefile) + singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile + + if DOCKERIZED: + # Perform the archive in the Archive docker container. (Typically in the development environment). + singlefile = base64.b64encode(singlefile).decode() + logger.debug(f'singlefile_to_archive sending to archive service: {url}') + singlefile, readability, screenshot = await request_archive(url, singlefile=singlefile) + else: + # JSON from readability-extractor + readability = dict() + try: + logger.debug(f'singlefile_to_archive extracting readability: {url}') + readability = await html_to_readability(singlefile, url) + except RuntimeError as e: + logger.error(f'Failed to extract readability from: {url}', exc_info=e) + + screenshot = None + try: + logger.debug(f'singlefile_to_archive creating screenshot: {url}') + screenshot = html_screenshot(singlefile) + except Exception as e: + logger.error(f'Failed to extract screenshot from: {url}', exc_info=e) + + logger.trace(f'singlefile_to_archive modeling: {url}') + archive: Archive = await model_archive_result(url, singlefile, readability, screenshot) + return archive diff --git a/modules/archive/models.py b/modules/archive/models.py index 0e5a311f..24877d1f 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -318,6 +318,11 @@ class Archive(Base, ModelHelper): def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile: return self.file_group.add_tag(tag_id_or_name, session) + @property + def location(self): + """The location where this Archive can be viewed in the UI.""" + return f'/archive/{self.id}' + class Domain(Base, ModelHelper): __tablename__ = 'domains' # plural to avoid conflict diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index 4defc429..d865d99b 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -3,7 +3,8 @@ from http import HTTPStatus import pytest -from modules.archive import lib +from modules.archive import lib, Archive +from wrolpi.test.common import skip_circleci def check_results(test_client, data, ids): @@ -227,3 +228,31 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory): request, response = test_client.get(f'/api/archive/domains') assert response.status_code == HTTPStatus.OK assert response.json['domains'] == [] + + +@skip_circleci +@pytest.mark.asyncio +async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form, + await_switches, events_history): + """Test converting a SingleFile from the SingleFile browser extension to an Archive.""" + singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url') + forms = [ + dict(name='url', value='https://example.com/form-url'), + dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents') + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/archive/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.OK + + await await_switches() + + archive, = test_session.query(Archive).all() + assert archive.singlefile_path.is_file(), 'Singlefile should have been saved.' + assert archive.readability_path.is_file(), 'Readability should be extracted.' + assert archive.readability_json_path.is_file(), 'Readability json should be extracted.' + assert archive.screenshot_path.is_file(), 'Screenshot should have been created' + assert archive.file_group.title == 'upload title', 'Title was not correct' + assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted' + events = list(events_history) + assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.' diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 130629d4..f6ed8d30 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -561,6 +561,35 @@ async def test_refresh_archives_index(test_session, make_files_structure): assert archive.file_group.indexed is True +@pytest.mark.asyncio +async def test_archive_meta(async_client, test_session, make_files_structure): + # The start of a typical singlefile html file. + singlefile_contents = ''' + +''' + + singlefile, *_ = make_files_structure({ + 'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents, + 'archive/example.com/2021-10-05-16-20-10_NA.png': None, + 'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents', + 'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '', + }) + + await files_lib.refresh_files() + + archive: Archive = test_session.query(Archive).one() + assert archive.singlefile_path == singlefile + + assert archive.file_group.author == 'A.B.C.' + assert archive.file_group.published_datetime == datetime(2021, 6, 20, 10, 0, tzinfo=pytz.utc) + + assert archive.file_group.title == 'The Headline' + assert archive.file_group.b_text == 'The Description' + assert archive.file_group.indexed is True + + @pytest.mark.asyncio async def test_refresh_archives_deleted_singlefile(test_session, make_files_structure, singlefile_contents_factory): """Removing a Singlefile file from a FileGroup makes that group no longer an Archive.""" diff --git a/modules/inventory/__init__.py b/modules/inventory/__init__.py index 6c9f8199..c0d70cf0 100644 --- a/modules/inventory/__init__.py +++ b/modules/inventory/__init__.py @@ -1,4 +1,3 @@ -import wrolpi from wrolpi.db import get_db_session from .api import inventory_bp from .inventory import logger, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES diff --git a/modules/map/lib.py b/modules/map/lib.py index f72ba914..9b65f932 100644 --- a/modules/map/lib.py +++ b/modules/map/lib.py @@ -1,5 +1,5 @@ -import asyncio import subprocess +from datetime import timedelta from pathlib import Path from typing import List @@ -8,9 +8,9 @@ from sqlalchemy.orm import Session from modules.map.models import MapFile from wrolpi import flags from wrolpi.api_utils import api_app -from wrolpi.cmd import SUDO_BIN +from wrolpi.cmd import SUDO_BIN, run_command from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config -from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp +from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp from wrolpi.db import optional_session, get_db_session from wrolpi.events import Events from wrolpi.vars import PROJECT_DIR, IS_RPI5 @@ -37,7 +37,7 @@ def is_pbf_file(pbf: Path) -> bool: def is_dump_file(path: Path) -> bool: - """Uses file command to check type of a file. Returns True if a file is a Postgresql dump file.""" + """Uses file command to check the type of file. Returns True if a file is a Postgresql dump file.""" cmd = ('/usr/bin/file', path) try: output = subprocess.check_output(cmd) @@ -188,24 +188,22 @@ async def run_import_command(*paths: Path) -> int: paths = ' '.join(str(i) for i in paths) # Run with sudo so renderd can be restarted. - cmd = f'{SUDO_BIN} {PROJECT_DIR}/scripts/import_map.sh {paths}' + cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', paths) import_logger.warning(f'Running map import command: {cmd}') - start = now() - proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE) + result = await run_command(cmd, debug=True) - stdout, stderr = await proc.communicate() - elapsed = now() - start - success = 'Successful' if proc.returncode == 0 else 'Unsuccessful' - import_logger.info(f'{success} import of {repr(paths)} took {timedelta_to_timestamp(elapsed)}') - if proc.returncode != 0: + success = 'Successful' if result.return_code == 0 else 'Unsuccessful' + import_logger.info( + f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}') + if result.return_code != 0: # Log all lines. Truncate long lines. - for line in stdout.decode().splitlines(): + for line in result.stdout.decode().splitlines(): import_logger.debug(line[:500]) - for line in stderr.decode().splitlines(): + for line in result.stderr.decode().splitlines(): import_logger.error(line[:500]) - raise ValueError(f'Importing map file failed with return code {proc.returncode}') + raise ValueError(f'Importing map file failed with return code {result.return_code}') - return int(elapsed.total_seconds()) + return result.elapsed @optional_session diff --git a/modules/map/test/test_api.py b/modules/map/test/test_api.py index 455b51bc..486f31ce 100644 --- a/modules/map/test/test_api.py +++ b/modules/map/test/test_api.py @@ -16,7 +16,7 @@ def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK): @mock.patch('modules.map.lib.is_pbf_file', lambda i: True) @mock.patch('modules.map.lib.is_dump_file', lambda i: True) -def test_status_and_import(test_client, test_session, make_files_structure, mock_create_subprocess_shell): +def test_status_and_import(test_client, test_session, make_files_structure, mock_run_command): """PBF files can be imported, and the status of the import can be monitored.""" pbf1, pbf2, dump1 = make_files_structure([ 'map/pbf/some-country.osm.pbf', @@ -32,11 +32,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock ] assert_map_file_status(test_client, expected) - with mock.patch('modules.map.lib.asyncio') as mock_asyncio: - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0) - body = {'files': [str(pbf1), ], } - request, response = test_client.post('/api/map/import', content=json.dumps(body)) - assert response.status_code == HTTPStatus.NO_CONTENT + body = {'files': [str(pbf1), ], } + request, response = test_client.post('/api/map/import', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT expected = [ {'imported': False, 'path': 'map/dump/dumpy.dump'}, @@ -45,11 +43,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock ] assert_map_file_status(test_client, expected) - with mock.patch('modules.map.lib.asyncio') as mock_asyncio: - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0) - body = {'files': [str(pbf1), str(pbf2)]} - request, response = test_client.post('/api/map/import', content=json.dumps(body)) - assert response.status_code == HTTPStatus.NO_CONTENT + body = {'files': [str(pbf1), str(pbf2)]} + request, response = test_client.post('/api/map/import', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT expected = [ {'imported': False, 'path': 'map/dump/dumpy.dump'}, @@ -58,11 +54,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock ] assert_map_file_status(test_client, expected) - with mock.patch('modules.map.lib.asyncio') as mock_asyncio: - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0) - body = {'files': [str(dump1), ]} - request, response = test_client.post('/api/map/import', content=json.dumps(body)) - assert response.status_code == HTTPStatus.NO_CONTENT + body = {'files': [str(dump1), ]} + request, response = test_client.post('/api/map/import', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT expected = [ {'imported': True, 'path': 'map/dump/dumpy.dump'}, @@ -74,7 +68,7 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock @mock.patch('modules.map.lib.is_pbf_file', lambda i: True) @mock.patch('modules.map.lib.is_dump_file', lambda i: True) -def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_create_subprocess_shell): +def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_run_command): """Multiple PBFs can be imported. Importing a second time overwrites the previous imports.""" pbf1, pbf2, pbf3, dump = make_files_structure([ 'map/pbf/country1.osm.pbf', @@ -83,11 +77,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s 'map/dumpy.dump', ]) - with mock.patch('modules.map.lib.asyncio') as mock_asyncio: - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0) - body = {'files': [str(pbf2), str(pbf1)]} - request, response = test_client.post('/api/map/import', content=json.dumps(body)) - assert response.status_code == HTTPStatus.NO_CONTENT + body = {'files': [str(pbf2), str(pbf1)]} + request, response = test_client.post('/api/map/import', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT # Two PBFs can be merged and imported. expected = [ @@ -98,11 +90,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s ] assert_map_file_status(test_client, expected) - with mock.patch('modules.map.lib.asyncio') as mock_asyncio: - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0) - body = {'files': [str(pbf3), ]} - request, response = test_client.post('/api/map/import', content=json.dumps(body)) - assert response.status_code == HTTPStatus.NO_CONTENT + body = {'files': [str(pbf3), ]} + request, response = test_client.post('/api/map/import', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT # New imports overwrite old imports. expected = [ @@ -113,12 +103,10 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s ] assert_map_file_status(test_client, expected) - with mock.patch('modules.map.lib.asyncio') as mock_asyncio: - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0) - # Can't mix PBF and dump. - body = {'files': [str(dump), ]} - request, response = test_client.post('/api/map/import', content=json.dumps(body)) - assert response.status_code == HTTPStatus.NO_CONTENT + # Can't mix PBF and dump. + body = {'files': [str(dump), ]} + request, response = test_client.post('/api/map/import', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT # Dump does not overwrite. expected = [ diff --git a/modules/map/test/test_lib.py b/modules/map/test/test_lib.py index be7c7121..25f3fcd2 100644 --- a/modules/map/test/test_lib.py +++ b/modules/map/test/test_lib.py @@ -22,7 +22,7 @@ def test_get_map_paths(test_directory, make_files_structure): @pytest.mark.asyncio -async def test_run_import_command(test_directory, mock_create_subprocess_shell): +async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result): """ Import map files. Files to import are checked for validity before importing. Any errors returned by asyncio.create_subprocess_shell are caught and reported. @@ -38,22 +38,14 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell): dump_file = test_directory / 'dumpy.dump' dump_file.touch() - with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \ - mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file: + with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file: # Run the import, it succeeds. - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell( - communicate_return=(b'out', b'error') - ) mock_is_pbf_file.return_value = True await lib.run_import_command(pbf_file) - with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \ - mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file: + with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file: # Run the import, it fails. - mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell( - communicate_return=(b'out', b'error'), - return_code=1, - ) + mock_run_command(run_command_bad_result) mock_is_pbf_file.return_value = True with pytest.raises(ValueError): await lib.run_import_command(pbf_file) @@ -102,7 +94,7 @@ def test_seconds_to_import_rpi5(size, expected): assert lib.seconds_to_import(size, True) == expected -def test_get_custom_map_directory(test_directory, test_wrolpi_config): +def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config): """Custom directory can be used for map directory.""" # Default location. assert lib.get_map_directory() == (test_directory / 'map') diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 9d75035f..f7052365 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -9,7 +9,6 @@ from sanic_ext.extensions.openapi import openapi from wrolpi.api_utils import json_response from wrolpi.common import logger, wrol_mode_check, get_media_directory, get_relative_to_media_directory -from wrolpi.downloader import download_manager from wrolpi.events import Events from wrolpi.schema import JSONErrorResponse from wrolpi.vars import PYTEST diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index f86b68ec..f991b4b4 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -14,7 +14,7 @@ from wrolpi.errors import APIError, ValidationError, RefreshConflict from wrolpi.vars import PYTEST from .. import schema from ..common import check_for_channel_conflicts -from ..errors import UnknownChannel, ChannelDirectoryConflict +from ..errors import UnknownChannel from ..lib import save_channels_config from ..models import Channel, Video diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index beaf5cc5..5301bbbc 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -57,7 +57,8 @@ def test_get_video(test_client, test_session, simple_channel, video_factory): @pytest.mark.asyncio -async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel, test_download_manager): +async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel, + test_download_manager): """A channel does not require a download frequency.""" # No downloads are scheduled. assert len(test_download_manager.get_downloads(test_session)) == 0 @@ -465,7 +466,7 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id) # Create recurring download which uses the Channel's directory. download = test_download_manager.recurring_download('https://example.com/1', 60, test_downloader.name, - destination=str(test_directory / 'videos/Channel Name')) + destination=str(test_directory / 'videos/Channel Name')) test_session.commit() save_channels_config() assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name') diff --git a/modules/videos/common.py b/modules/videos/common.py index 9345b729..f77dcffe 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -1,9 +1,7 @@ -import asyncio import json import os import pathlib import re -import shlex import subprocess import tempfile from datetime import timedelta @@ -16,7 +14,7 @@ from PIL import Image from sqlalchemy.orm import Session from wrolpi.captions import FFMPEG_BIN -from wrolpi.cmd import FFPROBE_BIN +from wrolpi.cmd import FFPROBE_BIN, run_command from wrolpi.common import logger, get_media_directory from wrolpi.dates import seconds_to_timestamp from wrolpi.db import get_db_session, get_db_curs @@ -213,28 +211,23 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict: ], } """ - cmd = f'{FFPROBE_BIN}' \ - f' -print_format json' \ - f' -loglevel quiet' \ - f' -show_streams' \ - f' -show_format' \ - f' -show_chapters' \ - f' {shlex.quote(str(video_path.absolute()))}' + cmd = (FFPROBE_BIN, + '-print_format', 'json', + '-loglevel', 'quiet', + '-show_streams', + '-show_format', + '-show_chapters', + video_path.absolute()) + + result = await run_command(cmd, timeout=60) + + if result.return_code != 0: + raise RuntimeError(f'Got non-zero exit code: {result.return_code}') try: - proc = await asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = await proc.communicate() + content = json.loads(result.stdout.decode().strip()) except Exception as e: - logger.error(f'Failed to run ffprobe json', exc_info=e) - raise - - if proc.returncode != 0: - raise RuntimeError(f'Got non-zero exit code: {proc.returncode}') - - try: - content = json.loads(stdout.decode().strip()) - except Exception as e: - logger.debug(stdout.decode()) + logger.debug(result.stdout.decode()) logger.error(f'Failed to load ffprobe json', exc_info=e) raise diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index c8ba1b86..072605c4 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -2,7 +2,6 @@ import json import pathlib import shutil from http import HTTPStatus -from itertools import zip_longest from typing import List from uuid import uuid4 @@ -14,6 +13,7 @@ from modules.videos.downloader import VideoDownloader, ChannelDownloader from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination from modules.videos.models import Channel, Video from wrolpi.api_utils import api_app +from wrolpi.cmd import CommandResult from wrolpi.downloader import DownloadFrequency, DownloadManager from wrolpi.files.models import FileGroup, Directory from wrolpi.vars import PROJECT_DIR @@ -208,22 +208,11 @@ def mock_video_prepare_filename(): @pytest.fixture def mock_video_process_runner(): with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_process_runner.return_value = (0, {'stdout': None, 'stderr': None}, None) + mock_process_runner.return_value = \ + CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0) yield mock_process_runner -@pytest.fixture -def assert_video_ids(test_session): - """Check that only the expected Videos are in the DB.""" - - def checker(expected_video_ids: List[int]): - video_ids = [i.id for i in test_session.query(Video).order_by(Video.id)] - for id_, expected in zip_longest(video_ids, expected_video_ids): - assert id_ == expected, f'Video ids were not as expected: {expected_video_ids=} != {video_ids=}' - - return checker - - @pytest.fixture def video_with_search_factory(test_session, test_directory, video_file_factory): """A factory that creates a Video record with an associated video File record.""" diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index d1f3e122..8848fdfe 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -416,13 +416,12 @@ class VideoDownloader(Downloader, ABC): '-o', video_path, url, ) - return_code, logs, _ = await self.process_runner(download_id, url, cmd, out_dir) + result = await self.process_runner(download, cmd, out_dir) + stdout = result.stdout.decode() + stderr = result.stderr.decode() - stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout'] - stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr'] - - if return_code != 0: - error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {return_code}' + if result.return_code != 0: + error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}' return DownloadResult( success=False, error=error, diff --git a/modules/videos/models.py b/modules/videos/models.py index 7a926c3e..040f12f2 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -65,19 +65,13 @@ class Video(ModelHelper, Base): except Exception as e: logger.error(f'{self} ffprobe_json is invalid', exc_info=e) - # TODO these are large objects. Can they be fetched on demand? - captions = self.file_group.d_text - comments = self.get_comments() - # Put live data in "video" instead of "data" to avoid confusion on the frontend. d['video'] = dict( - caption=captions, caption_files=self.caption_files, channel=channel, channel_id=self.channel_id, codec_names=codec_names, codec_types=codec_types, - comments=comments, comments_failed=self.comments_failed, description=self.file_group.c_text or self.get_video_description(), have_comments=self.have_comments, diff --git a/modules/videos/schema.py b/modules/videos/schema.py index 3909ed5a..de0aef45 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -155,6 +155,16 @@ class VideoResponse: next = ExtendedVideo +@dataclass +class VideoCommentsResponse: + comments: list[dict] + + +@dataclass +class VideoCaptionsResponse: + captions: str + + @dataclass class VideoSearchRequest: search_str: Optional[str] = None diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 8664e3be..3c8b788c 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -296,10 +296,10 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i await video_download_manager.wait_for_all_downloads() mock_video_process_runner.assert_called_once() - download_id, video_url, _, out_dir = mock_video_process_runner.call_args[0] + download, cmd, out_dir = mock_video_process_runner.call_args[0] download: Download = test_session.query(Download).one() - assert video_url == download.url + assert download.url in cmd assert test_session.query(Channel).one() video: Video = test_session.query(Video).one() diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index c9ae4323..85075223 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -28,6 +28,24 @@ def video_get(_: Request, video_id: int): return json_response({'file_group': video, 'prev': previous_video, 'next': next_video}) +@video_bp.get('/video//comments') +@openapi.description('Get Video comments') +@openapi.response(HTTPStatus.OK, schema.VideoCommentsResponse) +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +def video_get_comments(_: Request, video_id: int): + video = lib.get_video(video_id) + return json_response({'comments': video.get_comments()}) + + +@video_bp.get('/video//captions') +@openapi.description('Get Video captions') +@openapi.response(HTTPStatus.OK, schema.VideoCaptionsResponse) +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +def video_get_captions(_: Request, video_id: int): + video = lib.get_video(video_id) + return json_response({'captions': video.file_group.d_text}) + + @video_bp.post('/search') @openapi.definition( summary='Search Video titles and captions', diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 5f6c9d2a..71e955ce 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -1,14 +1,14 @@ import asyncio import random from datetime import timedelta -from typing import Tuple, Optional, List, Dict +from typing import Tuple, Optional, List import yt_dlp from sqlalchemy import or_ from sqlalchemy.orm import Session from modules.videos.models import Video -from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check +from wrolpi.common import logger, limit_concurrent, wrol_mode_check from wrolpi.dates import now from wrolpi.db import get_db_session, optional_session from wrolpi.downloader import download_manager @@ -17,7 +17,6 @@ from wrolpi.files.models import FileGroup from wrolpi.tags import tag_append_sub_select_where from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR from ..errors import UnknownVideo -from ..lib import save_channels_config logger.getChild(__name__) @@ -38,6 +37,15 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic return video, previous_video, next_video +def get_video(video_id: int) -> Video: + """ + Get a Video, with it's prev/next videos. Mark the Video as viewed. + """ + with get_db_session() as session: + video = Video.find_by_id(video_id, session=session) + return video + + VIDEO_ORDERS = { 'published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) ASC, LOWER(fg.primary_path) ASC', '-published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST,' diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index 6079a0ab..f820a682 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -187,3 +187,23 @@ async def test_wrol_mode(async_client, simple_channel, simple_video, wrol_mode_f await wrol_mode_fixture(False) assert not test_download_manager.stopped.is_set() + + +@pytest.mark.asyncio +async def test_api_video_extras(async_client, simple_channel, video_factory): + """Can fetch extra data about a video (comments/captions).""" + info_json = {'duration': 5, 'epoch': 12345, 'comments': [{'some': 'comments'}]} + video = video_factory(simple_channel.id, with_info_json=info_json, with_caption_file=True) + + request, response = await async_client.get(f'/api/videos/video/{video.id}/captions') + assert response.status_code == HTTPStatus.OK + assert response.json.get('captions') + + request, response = await async_client.get(f'/api/videos/video/{video.id}/comments') + assert response.status_code == HTTPStatus.OK + assert response.json.get('comments') + + request, response = await async_client.get('/api/videos/video/123/captions') + assert response.status_code == HTTPStatus.NOT_FOUND + request, response = await async_client.get('/api/videos/video/123/comments') + assert response.status_code == HTTPStatus.NOT_FOUND diff --git a/modules/zim/downloader.py b/modules/zim/downloader.py index a70a6a35..6ef1eb42 100644 --- a/modules/zim/downloader.py +++ b/modules/zim/downloader.py @@ -74,7 +74,7 @@ class KiwixZimDownloader(Downloader): zim_directory = lib.get_zim_directory() zim_directory.mkdir(parents=True, exist_ok=True) - output_path = await self.download_file(download.id, download.url, zim_directory) + output_path = await self.download_file(download, download.url, zim_directory) # Notify the maintainer if outdated Zim files are lying around. lib.flag_outdated_zim_files() diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 7c1d6492..076971e8 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -15,6 +15,7 @@ from modules.zim.errors import UnknownZim, UnknownZimSubscription from modules.zim.kiwix import KIWIX_CATALOG from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription from wrolpi import flags +from wrolpi.cmd import run_command from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate from wrolpi.db import get_db_session, optional_session, get_db_curs @@ -471,13 +472,12 @@ async def check_zim(path: pathlib.Path) -> int: @warning: Only performs the checksum check! """ - cmd = f'zimcheck -C {path.absolute()}' - proc = await asyncio.create_subprocess_shell(cmd) - stdout, stderr = await proc.communicate() - logger.debug(f'zimcheck returned {proc.returncode}') - if proc.returncode != 0 and stderr: - logger.debug(stderr) - return proc.returncode + cmd = ['zimcheck', '-C', path.absolute()] + result = await run_command(cmd) + logger.debug(f'zimcheck returned {result.return_code}') + if result.return_code != 0 and result.stderr: + logger.debug(result.stderr) + return result.return_code ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim') @@ -566,10 +566,9 @@ async def restart_kiwix(): logger.info('Restarting Kiwix serve') - cmd = f'sudo /usr/bin/systemctl restart wrolpi-kiwix.service' - proc = await asyncio.create_subprocess_shell(cmd) - stdout, stderr = await proc.communicate() - logger.debug(f'systemctl returned {proc.returncode}') - if proc.returncode != 0 and stderr: - logger.debug(stderr) - return proc.returncode + cmd = ('sudo', '/usr/bin/systemctl', 'restart' 'wrolpi-kiwix.service') + result = await run_command(cmd) + logger.debug(f'systemctl returned {result.return_code}') + if result.return_code != 0 and result.stderr: + logger.debug(result.stderr) + return result.return_code diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 4b1b2754..94c2aaea 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -12,6 +12,7 @@ from wrolpi.common import DownloadFileInfo, get_wrolpi_config from wrolpi.downloader import Download, import_downloads_config, save_downloads_config from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup +from wrolpi.test.common import skip_circleci @pytest.mark.asyncio @@ -182,6 +183,7 @@ def test_zim_all_entries(test_session, test_zim): assert len(entries) == 12 +@skip_circleci @pytest.mark.asyncio async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock, await_switches): diff --git a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh index 95bcb94a..a9e5a127 100755 --- a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh @@ -18,7 +18,7 @@ useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" sed -i 's/port =.*/port = 5432/' /etc/postgresql/15/main/postgresql.conf # Install Node console commands. -npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0 +npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0 # Put the latest WROLPi master in /opt/wrolpi. git clone -b master https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi diff --git a/scripts/install_debian_12.sh b/scripts/install_debian_12.sh index 3455c89d..3e6dc017 100755 --- a/scripts/install_debian_12.sh +++ b/scripts/install_debian_12.sh @@ -14,7 +14,7 @@ node -v npm -v # Install serve, and archiving tools. -single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0 +single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0 # Build React app in background job. cd /opt/wrolpi/app || exit 5 diff --git a/scripts/install_raspberrypios.sh b/scripts/install_raspberrypios.sh index f5c3c83d..d4bbfa27 100755 --- a/scripts/install_raspberrypios.sh +++ b/scripts/install_raspberrypios.sh @@ -14,7 +14,7 @@ node -v npm -v # Install serve, and archiving tools. -single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0 +single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0 # Build React app in background job. cd /opt/wrolpi/app || exit 5 diff --git a/wrolpi/admin.py b/wrolpi/admin.py index 659b2fb2..7fde727b 100644 --- a/wrolpi/admin.py +++ b/wrolpi/admin.py @@ -2,7 +2,7 @@ import asyncio import enum import subprocess -from wrolpi.cmd import NMCLI_BIN, CPUFREQ_INFO_BIN, SUDO_BIN, CPUFREQ_SET_BIN +from wrolpi.cmd import NMCLI_BIN, CPUFREQ_INFO_BIN, SUDO_BIN, CPUFREQ_SET_BIN, run_command from wrolpi.common import logger, WROLPI_CONFIG, get_warn_once, background_task from wrolpi.errors import ShutdownFailed from wrolpi.events import Events @@ -203,10 +203,8 @@ async def shutdown(reboot: bool = False, delay: int = 10): try: # Call shutdown (or reboot) with no delay now that we have slept. - cmd = f'{SUDO_BIN} shutdown {reboot} 0' - proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE) - stdout, stderr = await proc.communicate() + cmd = [SUDO_BIN, 'shutdown', 'reboot', '0'] + result = await run_command(cmd) except subprocess.CalledProcessError as e: if reboot: Events.send_shutdown_failed('Reboot failed') @@ -214,12 +212,12 @@ async def shutdown(reboot: bool = False, delay: int = 10): Events.send_shutdown_failed('Shutdown failed') raise ShutdownFailed() from e - if proc.returncode != 0: + if result.return_code != 0: if reboot: Events.send_shutdown_failed('Reboot failed') else: Events.send_shutdown_failed('Shutdown failed') - raise ShutdownFailed(f'Got return code {proc.returncode}') + raise ShutdownFailed(f'Got return code {result.return_code}') background_task(_()) if reboot: diff --git a/wrolpi/captions.py b/wrolpi/captions.py index 130f81ba..8fcd784f 100755 --- a/wrolpi/captions.py +++ b/wrolpi/captions.py @@ -60,7 +60,7 @@ def read_captions(caption_path: Path): logger.debug(f'Failed to parse caption file {caption_path}') -def extract_captions(path: pathlib.Path) -> Optional[str]: +def extract_captions(path: pathlib.Path) -> str | None: """Extract captions that are embedded in a video file.""" with tempfile.TemporaryDirectory() as directory: directory = pathlib.Path(directory) @@ -80,4 +80,4 @@ def extract_captions(path: pathlib.Path) -> Optional[str]: return captions # No captions could be extracted. - return + return None diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index 0ff68646..72429d02 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -1,10 +1,16 @@ """Paths for third-party commands that WROLPi may require.""" +import asyncio +import dataclasses import os +import pathlib import shutil +import tempfile +from asyncio import CancelledError from pathlib import Path +from time import time from typing import Optional -from wrolpi.common import logger +from wrolpi.common import logger, TRACE_LEVEL from wrolpi.vars import PYTEST, DOCKERIZED logger = logger.getChild(__name__) @@ -86,3 +92,86 @@ YT_DLP_BIN = which( FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe') FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg') JC_BIN = which('jc', '/usr/bin/jc') + + +@dataclasses.dataclass +class CommandResult: + return_code: int + cancelled: bool + stdout: bytes + stderr: bytes + elapsed: int + pid: int = None + + +TESTING_RUN_COMMAND_RESULT = None + + +async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None, + timeout: int = 600, debug: bool = False) -> CommandResult: + """Run a shell command, return the results (stdout/stderr/return code). + + :param debug: Enable debug logging (disabled by default because this is used by very frequent commands). + """ + if not isinstance(cmd, (list, tuple)): + raise RuntimeError('Command must be a list or tuple') + + if PYTEST and TESTING_RUN_COMMAND_RESULT: + logger.debug('run_command: returning mock result') + return TESTING_RUN_COMMAND_RESULT() # call the mock + + with tempfile.NamedTemporaryFile() as stdout_fh, tempfile.NamedTemporaryFile() as stderr_fh: + stdout_file, stderr_file = pathlib.Path(stdout_fh.name), pathlib.Path(stderr_fh.name) + cmd = tuple(str(i) for i in cmd) + if debug: + logger.debug(f'run_command: running ({timeout=}): {cmd=}') + elif logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'run_command: running ({timeout=}): {cmd=}') + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=stdout_fh, # Use stdout/stderr files to avoid buffer filling up. + stderr=stderr_fh, + cwd=cwd, + ) + pid = proc.pid + + cancelled = False + start = time() + try: + while True: + elapsed = int(time() - start) + if timeout and elapsed >= timeout: + logger.warning(f'run_command: timeout exceeded, killing... {cmd=}') + proc.kill() + await proc.wait() + break + + try: + # Wait for the process to finish. + await asyncio.wait_for(proc.wait(), timeout=1) + break + except asyncio.TimeoutError: + # Task is not done, keep waiting... + continue + except CancelledError as e: + logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e) + cancelled = True + proc.kill() + await proc.wait() + + elapsed = int(time() - start) + stdout = stdout_file.read_bytes() or b'' + stderr = stderr_file.read_bytes() or b'' + # Logs details of the call, but only if it took a long time or TRACE is enabled. + if debug: + logger.debug(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}') + elif logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}') + return CommandResult( + return_code=proc.returncode, + cancelled=cancelled, + stdout=stdout, + stderr=stderr, + elapsed=elapsed, + pid=pid, + ) diff --git a/wrolpi/common.py b/wrolpi/common.py index 01f1a827..10152d7b 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1732,6 +1732,15 @@ async def cancel_background_tasks(): await asyncio.gather(*BACKGROUND_TASKS) +async def await_background_tasks(): + """Awaits all background tasks, used only for testing.""" + if not PYTEST: + raise RuntimeError('await_background_tasks should only be used for testing') + + for background_task in BACKGROUND_TASKS.copy(): + await background_task + + def get_warn_once(message: str, logger__: logging.Logger, level=logging.ERROR): """Create a function that will report an error only once. @@ -1985,7 +1994,7 @@ def format_json_file(file: pathlib.Path, indent: int = 2): replace_file(file, content) -def format_html_string(html: str) -> str: +def format_html_string(html: str | bytes) -> str: if not html.strip(): raise RuntimeError('Refusing to format empty HTMl string.') diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index d33d47bb..5a4cd540 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -34,8 +34,9 @@ from sqlalchemy.orm import Session, sessionmaker import wrolpi.root_api # noqa from wrolpi import flags from wrolpi.api_utils import api_app +from wrolpi.cmd import CommandResult from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL -from wrolpi.common import logger +from wrolpi.common import logger, await_background_tasks as await_background_tasks_ from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts from wrolpi.dates import set_test_now @@ -202,6 +203,13 @@ async def await_switches(async_client): return await_switches_ +@pytest.fixture +async def await_background_tasks(async_client): + """Returns the `switches.await_switches` function, but primes the Sanic App to actually handle signals.""" + await async_client.get('/api') + return await_background_tasks_ + + @pytest.fixture async def test_download_manager( async_client, @@ -496,6 +504,32 @@ def mock_create_subprocess_shell() -> Callable: return mocker +@pytest.fixture +def mock_run_command(): + """Mock wrolpi.cmd's `run_command` using `TESTING_RUN_COMMAND_RESULT`.""" + default = CommandResult( + return_code=0, + cancelled=False, + stdout=b'mock_run_command fixture stdout', + stderr=b'mock_run_command fixture stderr', + elapsed=0, + pid=123, + ) + + with mock.patch('wrolpi.cmd.TESTING_RUN_COMMAND_RESULT') as mock_TESTING_RUN_COMMAND_RESULT: + mock_TESTING_RUN_COMMAND_RESULT.return_value = default + + def do_mock(new_result): + mock_TESTING_RUN_COMMAND_RESULT.return_value = new_result + + yield do_mock + + +@pytest.fixture +def run_command_bad_result() -> CommandResult: + return CommandResult(1, False, b'run_command_bad_result stdout', b'run_command_bad_result stderr', 0, 123) + + @pytest.fixture def events_history(async_client): """Give each test its own Events history.""" diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 4b334746..862a5a8e 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1,12 +1,10 @@ import asyncio import contextlib -import hashlib import inspect import logging import multiprocessing import os import pathlib -import re import tempfile import traceback import xml.etree.ElementTree as ET @@ -29,10 +27,10 @@ from sqlalchemy.sql import Delete from wrolpi import flags from wrolpi.api_utils import api_app, perpetual_signal -from wrolpi.cmd import which +from wrolpi.cmd import which, run_command, CommandResult from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \ - wrol_mode_enabled, background_task, get_wrolpi_config, get_absolute_media_path, timer, aiohttp_get, \ - get_download_info, trim_file_name + wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \ + get_download_info, trim_file_name, get_wrolpi_config from wrolpi.dates import TZDateTime, now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError @@ -300,61 +298,19 @@ class Downloader: def already_downloaded(self, *urls: List[str], session: Session = None): raise NotImplementedError() - async def process_runner(self, download_id: int, url: str, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path, - timeout: int = None, **kwargs) -> Tuple[int, dict, bytes]: + async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path, + timeout: int = None) -> CommandResult: """ Run a subprocess using the provided arguments. This process can be killed by the Download Manager. Global timeout takes precedence over the timeout argument, unless it is 0. (Smaller global timeout wins) """ - str_cmd = " ".join([str(i) for i in cmd]) - logger.info(f'{self} launching download process with args: {str_cmd}') - start = now() - # Use temporary files to prevent truncating of stdout/stderr. - with tempfile.NamedTemporaryFile() as stdout_file, tempfile.NamedTemporaryFile() as stderr_file: - stdout_file, stderr_file = pathlib.Path(stdout_file.name), pathlib.Path(stderr_file.name) - with stdout_file.open('wb') as stdout_fh, stderr_file.open('wb') as stderr_fh: - # stdout/stderr is written to real files. - proc = await asyncio.create_subprocess_exec(*cmd, - stdout=stdout_fh, - stderr=stderr_fh, - cwd=cwd, - **kwargs) - pid = proc.pid + if cwd and not cwd.is_dir(): + raise RuntimeError('cwd directory does not exist') - # Timeout can be any positive integer. Global download timeout takes precedence, unless it is 0. - # A timeout of 0 means the download will never be killed. - timeout = get_wrolpi_config().download_timeout or timeout or self.timeout - logger.debug(f'{self} launched download process {pid=} {timeout=} for {url}') - - while True: - # Calculate the seconds since the process started. - elapsed = int((now() - start).total_seconds()) - - # Check if the cancel event is set or if the timeout is reached - if download_manager.download_is_killed(download_id): - logger.warning(f'Killing download {pid=}, {elapsed} seconds elapsed.') - proc.kill() - await proc.wait() - break - if timeout and elapsed > timeout: - # TODO this logic should be handled by the Download Manager. - download_manager.kill_download(download_id) - continue - - try: - # Wait for the process to finish. - await asyncio.wait_for(proc.wait(), timeout=1) - break - except TimeoutError: - # Subprocess has not yet finished, loop again and wait for it to finish. - continue - - # Read stdout/stderr files. - stdout, stderr = stdout_file.read_bytes(), stderr_file.read_bytes() - stdout_len, stderr_len = len(stdout) if stdout else 0, len(stderr) if stderr else 0 - logger.debug(f'Download exited with {proc.returncode} and stdout={stdout_len} stderr={stderr_len}') - return proc.returncode, dict(stdout=stdout, stderr=stderr), stdout + timeout = get_wrolpi_config().download_timeout or timeout or self.timeout + coro = run_command(cmd, cwd=cwd, timeout=timeout) + return await self.cancel_wrapper(coro, download) @staticmethod async def cancel_wrapper(coro: Coroutine, download: Download): @@ -404,8 +360,8 @@ class Downloader: return None - async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True, - check_for_md5: bool = True) -> pathlib.Path: + async def download_file(self, download: Download, url: str, destination: pathlib.Path, check_for_meta4: bool = True) \ + -> pathlib.Path: from wrolpi.files.lib import glob_shared_stem if not ARIA2C_PATH: @@ -439,18 +395,11 @@ class Downloader: '-M', meta4_path, ) - return_code, logs, stdout = await self.process_runner( - download_id, - url, # Log the original URL. - cmd, - destination, - ) + result = await self.process_runner(download, cmd, destination) + error = result.stderr.decode() - error = logs.get('stderr') or logs.get('stdout') - error = error.decode() if error else error - - if return_code != 0: - raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}') + if result.return_code != 0: + raise DownloadError(f'{error}\n\nFileDownloader failed with return code {result.return_code}') if not output_path.is_file(): raise DownloadError(f'{error}\n\nOutput file not found: {output_path}') diff --git a/wrolpi/events.py b/wrolpi/events.py index ce61378a..17810936 100644 --- a/wrolpi/events.py +++ b/wrolpi/events.py @@ -92,6 +92,10 @@ class Events: def send_config_save_failed(message: str = None): send_event('config_save_failed', message, subject='configs') + @classmethod + def send_archive_uploaded(cls, message: str = None, url: str = None): + send_event('upload_archive', message, subject='upload', url=url) + def log_event(event: str, message: str = None, action: str = None, subject: str = None): log = f'{event=}' diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 331be78a..8742e1ef 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -45,6 +45,9 @@ async def get_file(_: Request, body: schema.FileRequest): file = lib.get_file_dict(body.file) except FileNotFoundError: raise InvalidFile() + + # TODO ensure that this is being called during file previews and preview on page load + background_task(lib.set_file_viewed(get_media_directory() / body.file)) return json_response({'file': file}) @@ -102,7 +105,7 @@ async def post_search_files(_: Request, body: schema.FilesSearchRequest): with timer('Searching all files', 'info', logger__=logger): file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model, body.tag_names, body.headline, body.months, body.from_year, body.to_year, - body.any_tag) + body.any_tag, body.order) return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total))) diff --git a/wrolpi/files/downloader.py b/wrolpi/files/downloader.py index 61243918..9c18fd19 100644 --- a/wrolpi/files/downloader.py +++ b/wrolpi/files/downloader.py @@ -1,5 +1,4 @@ import pathlib -import pathlib import traceback from abc import ABC @@ -52,7 +51,7 @@ class FileDownloader(Downloader, ABC): output_path = None try: - output_path = await self.download_file(download.id, download.url, destination) + output_path = await self.download_file(download, download.url, destination) fg = await upsert_file(output_path, tag_names=download.tag_names) location = fg.location return DownloadResult( diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index edcd4770..54f10291 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -90,6 +90,18 @@ def get_file_dict(file: str) -> Dict: return _get_file_dict(media_directory / file) +@optional_session +async def set_file_viewed(file: pathlib.Path, session: Session = None): + """Change FileGroup.viewed to the current datetime.""" + try: + fg = FileGroup.find_by_path(file, session) + except UnknownFile: + fg = FileGroup.from_paths(session, file) + fg.do_model(session) + fg.set_viewed() + session.commit() + + @cachetools.func.ttl_cache(10_000, 30.0) def _get_directory_dict(directory: pathlib.Path, directories_cache: str, # Used to cache by requested directories. @@ -774,7 +786,7 @@ async def search_directories_by_name(name: str, excluded: List[str] = None, limi def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None, tag_names: List[str] = None, headline: bool = False, months: List[int] = None, - from_year: int = None, to_year: int = None, any_tag: bool = False) -> \ + from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None) -> \ Tuple[List[dict], int]: """Search the FileGroup table. @@ -792,6 +804,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] @param tag_names: A list of tag names. @param headline: Includes Postgresql headline if True. @param months: A list of integers representing the index of the month of the year, starting at 1. + @param order: Used to change results from most relevant to recently viewed. """ params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%') wheres = [] @@ -823,6 +836,15 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] else: headline = '' + if order == 'viewed': + order_by = 'viewed DESC NULLS LAST, 1 ASC' + elif order == '-viewed': + order_by = 'viewed ASC NULLS LAST, 1 ASC' + + if order and not search_str: + # Only filter out unviewed files if search_str is not provided. + wheres.append('viewed IS NOT NULL') + wheres = '\n AND '.join(wheres) selects = f"{', '.join(selects)}, " if selects else "" join = '\n'.join(joins) diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index b4d4ffc4..990657e3 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -1,7 +1,6 @@ import copy import pathlib import shutil -import urllib.parse from datetime import datetime from functools import singledispatchmethod from typing import List, Type, Optional, Iterable @@ -11,7 +10,7 @@ from sqlalchemy import types from sqlalchemy.orm import deferred, relationship, Session from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \ - get_relative_to_media_directory, unique_by_predicate + unique_by_predicate, TRACE_LEVEL from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime from wrolpi.db import optional_session from wrolpi.downloader import Download @@ -141,8 +140,13 @@ class FileGroup(ModelHelper, Base): return self.title return self.primary_path.name - def set_viewed(self): - self.viewed = now() + def set_viewed(self, viewed: datetime = None) -> datetime: + """ + :param viewed: Used only for testing! + :return: The datetime that was set. + """ + self.viewed = viewed or now() + return self.viewed @optional_session def set_tags(self, tag_names_or_ids: Iterable[str | int], session: Session = None): @@ -308,7 +312,6 @@ class FileGroup(ModelHelper, Base): raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem: {paths[0]}') existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all() - logger.debug(f'FileGroup.from_paths: {len(existing_groups)}') if len(existing_groups) == 0: # These paths have not been used previously, create a new FileGroup. file_group = FileGroup() @@ -336,7 +339,8 @@ class FileGroup(ModelHelper, Base): file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths)) file_group.size = sum(i.stat().st_size for i in paths) file_group.mimetype = get_mimetype(file_group.primary_path) - logger.debug(f'FileGroup.from_paths: {file_group}') + if logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'FileGroup.from_paths: {file_group}') return file_group @@ -353,6 +357,14 @@ class FileGroup(ModelHelper, Base): file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none() return file_group + @staticmethod + @optional_session + def find_by_path(path, session) -> 'FileGroup': + file_group = FileGroup.get_by_path(path, session) + if not file_group: + raise UnknownFile(f'Unable to find FileGroup with path {path}') + return file_group + @property def tag_names(self) -> List[str]: return [i.tag.name for i in self.tag_files] diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py index b303cbd9..ad9a2101 100644 --- a/wrolpi/files/schema.py +++ b/wrolpi/files/schema.py @@ -32,6 +32,7 @@ class FilesSearchRequest: from_year: Optional[int] = None to_year: Optional[int] = None any_tag: bool = False + order: Optional[str] = None def __post_init__(self): if self.any_tag and self.tag_names: diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index c8bc6dc1..54538e58 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -1,5 +1,6 @@ import hashlib import json +from datetime import datetime, timezone from http import HTTPStatus from unittest import mock @@ -160,6 +161,42 @@ async def test_delete_wrol_mode(async_client, wrol_mode_fixture): assert response.json['code'] == 'WROL_MODE_ENABLED' +@pytest.mark.asyncio +async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory): + """File search can return the most recently viewed files.""" + video_file_factory(test_directory / 'foo.mp4'), video_file_factory(test_directory / 'bar.mp4') + baz = (test_directory / 'baz.txt') + baz.write_text('baz contents') + await lib.refresh_files() + + # baz is most recently viewed, foo has not been viewed. + bar, baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all() + bar.set_viewed(datetime(2000, 1, 1, 1, 1, 1, tzinfo=timezone.utc)) + baz.set_viewed(datetime(2000, 1, 1, 1, 1, 2, tzinfo=timezone.utc)) + test_session.commit() + + # Results match the viewed order. foo is not viewed. + body = dict(order='viewed') + request, response = await async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['file_groups'], response.json + assert [i['name'] for i in response.json['file_groups']] == ['baz.txt', 'bar.mp4'] + + # Results match the reverse viewed order. foo is not viewed. + body = dict(order='-viewed') + request, response = await async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['file_groups'], response.json + assert [i['name'] for i in response.json['file_groups']] == ['bar.mp4', 'baz.txt'] + + # foo is included because `search_str` has a value. + body = dict(search_str='foo', order='viewed') + request, response = await async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['file_groups'], response.json + assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4'] + + def test_files_search(test_session, test_client, make_files_structure, assert_files_search): # You can search an empty directory. assert_files_search('nothing', []) @@ -757,7 +794,7 @@ async def test_delete_directory_recursive(test_session, test_directory, make_fil @pytest.mark.asyncio -async def test_get_file(test_session, async_client, test_directory, make_files_structure): +async def test_get_file(test_session, async_client, test_directory, make_files_structure, await_background_tasks): """Can get info about a single file.""" make_files_structure({'foo/bar.txt': 'foo contents'}) await lib.refresh_files() @@ -768,6 +805,13 @@ async def test_get_file(test_session, async_client, test_directory, make_files_s assert response.json['file'] assert response.json['file']['path'] == 'foo/bar.txt' + # Wait for `viewed` to be set in background task. + await await_background_tasks() + + fg, = test_session.query(FileGroup).all() + assert fg.primary_path == test_directory / 'foo/bar.txt' + assert isinstance(fg.viewed, datetime) + @pytest.mark.asyncio async def test_ignore_directory(test_session, async_client, test_directory, make_files_structure, test_wrolpi_config, diff --git a/wrolpi/files/test/test_scrape_downloader.py b/wrolpi/files/test/test_scrape_downloader.py index 3471fcb1..cb433deb 100644 --- a/wrolpi/files/test/test_scrape_downloader.py +++ b/wrolpi/files/test/test_scrape_downloader.py @@ -30,7 +30,8 @@ async def fake_file_do_download(*a, **kwargs): return DownloadResult(success=True) -async def test_scrape_html_downloader(test_directory, test_session, test_download_manager, assert_download_urls, await_switches): +async def test_scrape_html_downloader(test_directory, test_session, test_download_manager, assert_download_urls, + await_switches): """User can scrape for PDF files.""" test_download_manager.register_downloader(ScrapeHTMLDownloader()) test_download_manager.register_downloader(FileDownloader()) diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 3b97fff1..edfe044c 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -370,4 +370,3 @@ class SearchFileEstimateRequest: @dataclass class SearchOtherEstimateRequest: tag_names: List[str] = field(default_factory=list) - diff --git a/wrolpi/status.py b/wrolpi/status.py index 61f145be..b903d388 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -14,7 +14,7 @@ from typing import List, Optional, Dict import jc from wrolpi.api_utils import api_app, perpetual_signal -from wrolpi.cmd import which +from wrolpi.cmd import which, run_command from wrolpi.common import logger, get_warn_once, unique_by_predicate, partition, TRACE_LEVEL from wrolpi.dates import now @@ -86,14 +86,11 @@ class IostatInfo: async def get_iostat_stats() -> IostatInfo: try: - proc = await asyncio.create_subprocess_shell( - 'iostat 3 2', # 3 second interval to gather more data, get 2 count. - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + result = await run_command( + ('iostat', '3', '2'), # 3 second interval to gather more data, get 2 count. ) - stdout, stderr = await proc.communicate() - if stdout and proc.returncode == 0: - iostat_stats = jc.parse('iostat', stdout.decode(), quiet=True) + if result.stdout and result.return_code == 0: + iostat_stats = jc.parse('iostat', result.stdout.decode(), quiet=True) if iostat_stats: cpu_statuses, _ = partition(lambda i: i['type'] == 'cpu', iostat_stats) _, cpu_stats = cpu_statuses # Use the second status because the first is a lie. @@ -216,21 +213,16 @@ async def get_cpu_stats() -> CPUInfo: return info -PS_CMD = 'ps aux --sort=-%cpu --cols=512' -IGNORED_PROCESS_COMMANDS = {PS_CMD, ''} +PS_CMD = ('ps', 'aux', '--sort=-%cpu', '--cols=512') +IGNORED_PROCESS_COMMANDS = {' '.join(PS_CMD), ''} async def get_processes_stats() -> List[ProcessInfo]: - proc = await asyncio.create_subprocess_shell( - PS_CMD, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + result = await run_command(PS_CMD) # Do not include `ps` in the top processes. - ps_pid = proc.pid - stdout, stderr = await proc.communicate() + ps_pid = result.pid - ps_info = jc.parse('ps', stdout.decode(), quiet=True) + ps_info = jc.parse('ps', result.stdout.decode(), quiet=True) processes = sorted(ps_info, key=lambda i: i['cpu_percent'], reverse=True) processes = [i for i in processes if i['pid'] != ps_pid and i['cpu_percent'] > 1 @@ -553,15 +545,11 @@ async def get_power_stats() -> PowerStats: logger.trace(f'get_power_stats called') try: - proc = await asyncio.create_subprocess_shell( - 'dmesg', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, stderr = await proc.communicate() - if stdout and proc.returncode == 0: - under_voltage = b' Undervoltage detected' in stdout - over_current = b' over-current change ' in stdout + cmd = ('dmesg',) + result = await run_command(cmd) + if result.stdout and result.return_code == 0: + under_voltage = b' Undervoltage detected' in result.stdout + over_current = b' over-current change ' in result.stdout return PowerStats( under_voltage=under_voltage, over_current=over_current, diff --git a/wrolpi/test/test_cmd.py b/wrolpi/test/test_cmd.py index 43613ab0..e38030e1 100644 --- a/wrolpi/test/test_cmd.py +++ b/wrolpi/test/test_cmd.py @@ -1,3 +1,4 @@ +import asyncio import os from pathlib import Path @@ -20,3 +21,46 @@ def test_which(test_directory): assert cmd.which('asdf', str(foo)) is None os.chmod(foo, 0o777) assert cmd.which('asdf', str(foo)) == foo + + +TEST_SCRIPT = '''#!/bin/sh +echo stdout +echo stderr >&2 +''' + + +async def test_run_command(test_directory): + """Can read both stdin and stderr. Can get return code.""" + (test_directory / 'script.sh').write_text(TEST_SCRIPT) + result = await cmd.run_command(('sh', 'script.sh'), cwd=test_directory) + assert result.return_code == 0 + assert result.cancelled is False + assert result.stdout == b'stdout\n' + assert result.stderr == b'stderr\n' + assert result.elapsed == 0 + + +async def test_run_command_timeout(test_directory): + """Timeout is obeyed.""" + result = await cmd.run_command(('sleep', '10'), timeout=2) + assert result.return_code == -9 + assert result.cancelled is False + assert result.stdout == b'' + assert result.stderr == b'' + assert result.elapsed >= 2 + + +async def test_run_command_cancel(test_directory): + """Run a command, cancel it and test that it was cancelled.""" + task = asyncio.create_task(cmd.run_command(('sleep', '10'))) + try: + await asyncio.wait_for(task, timeout=1) + except TimeoutError: + task.cancel() + + result = await task + assert result.return_code == -9 + assert result.cancelled is True + assert result.stdout == b'' + assert result.stderr == b'' + assert result.elapsed == 1 diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index bafc2331..e5f0105b 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -1,7 +1,6 @@ import asyncio import json import pathlib -import tempfile from abc import ABC from datetime import datetime from http import HTTPStatus @@ -303,26 +302,26 @@ async def test_process_runner_timeout(async_client, test_session, test_directory # Sleep for 8 seconds really takes 8 seconds. start = datetime.now() - await downloader.process_runner(1, 'https://example.com', ('sleep', '8'), test_directory, timeout=10) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=10) elapsed = datetime.now() - start assert elapsed.total_seconds() >= 8 - # Default timeout is obeyed. + # Download.timeout is obeyed. start = datetime.now() - await downloader.process_runner(1, 'https://example.com', ('sleep', '8'), test_directory) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory) elapsed = datetime.now() - start assert 3 < elapsed.total_seconds() < 5 # One-off timeout is obeyed. start = datetime.now() - await downloader.process_runner(2, 'https://example.com', ('sleep', '8'), test_directory, timeout=1) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=1) elapsed = datetime.now() - start assert 1 < elapsed.total_seconds() < 3 # Global timeout is obeyed. get_wrolpi_config().download_timeout = 3 start = datetime.now() - await downloader.process_runner(3, 'https://example.com', ('sleep', '8'), test_directory) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory) elapsed = datetime.now() - start assert 2 < elapsed.total_seconds() < 5 @@ -341,34 +340,30 @@ exit 123 @pytest.mark.asyncio async def test_process_runner_output(async_client, test_directory, test_downloader): - with tempfile.TemporaryDirectory() as directory: - directory = pathlib.Path(directory) - script = directory / 'echo_script.sh' - script.write_text(GOOD_SCRIPT) - cmd = ('/bin/bash', script) - return_code, logs, stdout = await test_downloader.process_runner( - 1, - 'https://example.com', - cmd, - directory, - timeout=1 - ) - assert return_code == 0 - assert logs['stdout'] == b'standard output\n' - assert logs['stderr'] == b'standard error\n' + script = test_directory / 'echo_script.sh' + script.write_text(GOOD_SCRIPT) + cmd = ('/bin/bash', script) + result = await test_downloader.process_runner( + Download(), + cmd, + test_directory, + timeout=1 + ) + assert result.return_code == 0 + assert result.stdout == b'standard output\n' + assert result.stderr == b'standard error\n' - script.write_text(BAD_SCRIPT) - cmd = ('/bin/bash', script) - return_code, logs, stdout = await test_downloader.process_runner( - 1, - 'https://example.com', - cmd, - directory, - timeout=1 - ) - assert return_code == 123 - assert logs['stdout'] == b'bad standard output\n' - assert logs['stderr'] == b'bad standard error\n' + script.write_text(BAD_SCRIPT) + cmd = ('/bin/bash', script) + result = await test_downloader.process_runner( + Download(), + cmd, + test_directory, + timeout=1 + ) + assert result.return_code == 123 + assert result.stdout == b'bad standard output\n' + assert result.stderr == b'bad standard error\n' @pytest.mark.asyncio diff --git a/wrolpi/test/test_rss.py b/wrolpi/test/test_rss.py index 7e911fe3..0981678e 100644 --- a/wrolpi/test/test_rss.py +++ b/wrolpi/test/test_rss.py @@ -104,7 +104,7 @@ async def test_rss_downloader_filter_titles(test_session): rss_downloader = RSSDownloader() entries = [ - dict(link='https://example.com/a', title='A'), # Case should be ignored. + dict(link='https://example.com/a', title='A'), # Case should be ignored. dict(link='https://example.com/b', title='b'), dict(link='https://example.com/c', title='c'), ]