diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 7a7c0f4d..4b009aa8 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -1,5 +1,5 @@ import React, {useContext, useEffect, useState} from "react"; -import {Card, Container, Image, Input, Pagination, Responsive, TableCell, TableRow} from 'semantic-ui-react'; +import {Card, Container, IconGroup, Image, Input, Pagination, Responsive, TableCell, TableRow} from 'semantic-ui-react'; import {Link, NavLink} from "react-router-dom"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks"; @@ -734,6 +734,24 @@ export function DarkModeToggle() { } +export function HotspotStatusIcon() { + const {on, setHotspot} = useHotspot(); + const toggleHotspot = (e) => { + e.preventDefault(); + if (on != null) { + setHotspot(!on); + } + } + + return + + + {on === false && } + {on === null && } + + +} + export function useTitle(title) { const documentDefined = typeof document !== 'undefined'; const originalTitle = React.useRef(documentDefined ? document.title : null); diff --git a/app/src/components/Nav.js b/app/src/components/Nav.js index dc05c5ed..f981827f 100644 --- a/app/src/components/Nav.js +++ b/app/src/components/Nav.js @@ -3,7 +3,7 @@ import {NavLink} from "react-router-dom"; import {Dropdown, Menu, Responsive} from "semantic-ui-react"; import {StatusContext, ThemeContext} from "../contexts/contexts"; import Icon from "semantic-ui-react/dist/commonjs/elements/Icon"; -import {DarkModeToggle} from "./Common"; +import {DarkModeToggle, HotspotStatusIcon} from "./Common"; const responsiveWidth = 500; @@ -63,6 +63,10 @@ function MenuLink(props) { } } +function NavIcon(props) { + return
{props.children}
+} + export function NavBar() { const {status} = useContext(StatusContext); const wrol_mode = status ? status['wrol_mode'] : null; @@ -81,17 +85,15 @@ export function NavBar() { ) })} -
- -
+ + {rightLinks.map((link) => )}
{/*Show the menu items in a dropdown when on mobile*/} -
- -
+ + {collapsedLinks.map((link) => diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 44943dad..72aa94a8 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -169,7 +169,7 @@ function Statistics() { } const videoNames = [ - {key: 'videos', label: 'Downloaded Videos'}, + {key: 'videos', label: 'Videos'}, {key: 'favorites', label: 'Favorite Videos'}, {key: 'sum_size', label: 'Total Size'}, {key: 'max_size', label: 'Largest Video'}, diff --git a/app/src/components/admin/Status.js b/app/src/components/admin/Status.js index 54d83b83..7ce2c4af 100644 --- a/app/src/components/admin/Status.js +++ b/app/src/components/admin/Status.js @@ -1,10 +1,10 @@ -import {Header, Loader, Progress, Statistic, StatisticGroup} from "../Theme"; -import React from "react"; +import {Header, Progress, Statistic, StatisticGroup} from "../Theme"; +import React, {useContext} from "react"; import {humanBandwidth, humanFileSize, LoadStatistic} from "../Common"; -import {getStatus} from "../../api"; -import {Container, Divider} from "semantic-ui-react"; +import {Divider} from "semantic-ui-react"; import {ProgressPlaceholder} from "../Placeholder"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; +import {StatusContext} from "../../contexts/contexts"; function DriveInfo({used, size, percent, mount}) { let color; @@ -102,75 +102,64 @@ export function CPUUsageProgress({value, label}) { return } -export class Status extends React.Component { +export function Status() { + const {status} = useContext(StatusContext); - constructor(props) { - super(props); - this.state = { - status: null, - } - this.fetchStatus = this.fetchStatus.bind(this); + let percent; + let cores; + let temperature; + let high_temperature; + let critical_temperature; + let minute_1; + let minute_5; + let minute_15; + let bandwidth; + let drives = []; + + if (status && status['cpu_info']) { + const {cpu_info, load} = status; + percent = cpu_info['percent']; + cores = cpu_info['cores']; + temperature = cpu_info['temperature']; + high_temperature = cpu_info['high_temperature']; + critical_temperature = cpu_info['critical_temperature']; + + minute_1 = load['minute_1']; + minute_5 = load['minute_5']; + minute_15 = load['minute_15']; + + drives = status['drives']; + bandwidth = status['bandwidth']; } - async componentDidMount() { - await this.fetchStatus(); - this.intervalId = setInterval(this.fetchStatus, 1000 * 5); - } + return <> + - componentWillUnmount() { - clearInterval(this.intervalId); - } + + - fetchStatus = async () => { - try { - let status = await getStatus(); - this.setState({status: status}); - } catch (e) { - console.error(e); - this.setState({status: null}); - } - } + - render() { - if (this.state.status) { - const {cpu_info, load, drives, bandwidth} = this.state.status; - const {minute_1, minute_5, minute_15} = load; - const {percent, cores, temperature, high_temperature, critical_temperature} = cpu_info; + + + + + - return <> - + - - +
Bandwidth
+ {bandwidth ? + bandwidth.map(i => ) : + } - + - - - - - - - - -
Bandwidth
- {bandwidth ? - bandwidth.map(i => ) : - } - - - -
Drive Usage
- {drives.map((drive) => )} - - } - - return - Loading system status - - } +
Drive Usage
+ {drives.map((drive) => )} + } diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 437d6e46..6ff24c54 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -1,4 +1,4 @@ -import {useEffect, useState} from "react"; +import {useContext, useEffect, useRef, useState} from "react"; import { favoriteVideo, fetchDomains, @@ -27,6 +27,7 @@ import { import {createSearchParams, useSearchParams} from "react-router-dom"; import {toast} from "react-semantic-toasts"; import {enumerate, humanFileSize, secondsToFullDuration} from "../components/Common"; +import {StatusContext} from "../contexts/contexts"; const calculatePage = (offset, limit) => { return offset && limit ? Math.round((offset / limit) + 1) : 1; @@ -36,6 +37,25 @@ const calculateTotalPages = (total, limit) => { return total && limit ? Math.round(total / limit) + 1 : 1; } +export const useRecurringTimeout = (callback, delay) => { + const timer = useRef(null); + + useEffect(() => { + const repeat = async () => { + // Recursively call the callback. + await callback(); + timer.current = window.setTimeout(repeat, delay); + } + + // Call the repeating function instantly to fetch data fast. + repeat(); + + // Clear the timeout on unload. + return () => clearTimeout(timer.current); + }, [delay]) +} + + export const useDomains = () => { const [domains, setDomains] = useState(null); const [total, setTotal] = useState(null); @@ -441,34 +461,24 @@ export const useBrowseFiles = () => { export const useHotspot = () => { const [on, setOn] = useState(null); - - const fetchHotspotStatus = async () => { - let status; - try { - status = await getHotspotStatus(); - } catch (e) { - console.error(e); - } - if (status === 'connected') { - // Hotspot is on. - setOn(true); - } else if (status === 'disconnected' || status === 'unavailable') { - // Hotspot can be turned on. - setOn(false); - } else { - // Hotspot is not supported. API is probably running in a Docker container. - setOn(null); - } - } + const {status} = useContext(StatusContext); useEffect(() => { - fetchHotspotStatus(); - }, []); + if (status && status['hotspot_status']) { + const {hotspot_status} = status; + if (hotspot_status === 'connected') { + setOn(true); + } else if (hotspot_status === 'disconnected' || hotspot_status === 'unavailable') { + setOn(false); + } else { + setOn(null); + } + } + }, [status]); const localSetHotspot = async (on) => { setOn(null); await setHotspot(on); - await fetchHotspotStatus(); } return {on, setOn, setHotspot: localSetHotspot}; @@ -488,13 +498,7 @@ export const useDownloads = () => { } } - useEffect(() => { - fetchDownloads(); - const interval = setInterval(() => { - fetchDownloads(); - }, 1000 * 3); - return () => clearInterval(interval); - }, []); + useRecurringTimeout(fetchDownloads, 3 * 1000); return {onceDownloads, recurringDownloads, fetchDownloads} } @@ -616,12 +620,7 @@ export const useSettings = () => { export const useSettingsInterval = () => { const {settings, fetchSettings} = useSettings(); - useEffect(() => { - const interval = setInterval(() => { - fetchSettings(); - }, 1000 * 3); - return () => clearInterval(interval); - }, []); + useRecurringTimeout(fetchSettings(), 1000 * 3); return {settings, fetchSettings}; } @@ -644,27 +643,13 @@ export const useStatus = () => { } } - useEffect(() => { - fetchStatus(); - const interval = setInterval(() => { - // console.log('fetch'); - fetchStatus(); - }, 1000 * 5); - return () => clearInterval(interval); - }, []); - return {status, fetchStatus} } export const useStatusInterval = () => { const {status, fetchStatus} = useStatus(); - useEffect(() => { - const interval = setInterval(() => { - fetchStatus(); - }, 1000 * 3); - return () => clearInterval(interval); - }, []); + useRecurringTimeout(fetchStatus, 1000 * 3); return {status, fetchStatus}; } diff --git a/etc/ubuntu20.04/wrolpi-api.service b/etc/ubuntu20.04/wrolpi-api.service index 212b76ec..069696c1 100644 --- a/etc/ubuntu20.04/wrolpi-api.service +++ b/etc/ubuntu20.04/wrolpi-api.service @@ -5,7 +5,7 @@ PartOf=wrolpi.target [Service] Environment=PYTHONUNBUFFERED=1 -ExecStart=/usr/bin/python3 /opt/wrolpi/main.py api +ExecStart=/usr/bin/python3 /opt/wrolpi/main.py -v api Group=wrolpi User=wrolpi WorkingDirectory=/opt/wrolpi/ diff --git a/main.py b/main.py index b27010ab..585becf0 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,6 @@ #! /usr/bin/env python3 import argparse import logging -import os import sys import pytz @@ -12,6 +11,7 @@ from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, wrol_mode_enabled from wrolpi.dates import set_timezone from wrolpi.downloader import download_manager +from wrolpi.files.lib import cancel_refresh_tasks from wrolpi.root_api import api_app from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST from wrolpi.version import get_version_string @@ -214,10 +214,11 @@ async def bandwidth_worker(app: Sanic): @root_api.api_app.signal(Event.SERVER_SHUTDOWN_BEFORE) @limit_concurrent(1) -def handle_server_shutdown(*args, **kwargs): +async def handle_server_shutdown(*args, **kwargs): """Stop downloads when server is shutting down.""" if not PYTEST: download_manager.stop() + await cancel_refresh_tasks() if __name__ == '__main__': diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index e342854b..4ba1006b 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -1,4 +1,3 @@ -import itertools import json import pathlib from abc import ABC @@ -63,10 +62,10 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session): if any(lib.is_singlefile_file(i.path) for i in group): archive_groups[stem] = group - count = itertools.cycle(range(101)) for stem, group in archive_groups.items(): readability_file, singlefile_file, readability_json_file, readability_txt_file, screenshot_file = \ match_archive_files(group) + archive: Archive = Archive.find_by_path(singlefile_file.path, session) if not archive: archive = Archive(singlefile_file=singlefile_file) @@ -78,16 +77,16 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session): if archive.readability_file: archive.readability_file.associated = True - archive.readability_file.do_index() + archive.readability_file.do_stats() if archive.readability_json_file: archive.readability_json_file.associated = True - archive.readability_json_file.do_index() + archive.readability_json_file.do_stats() if archive.readability_txt_file: archive.readability_txt_file.associated = True - archive.readability_txt_file.do_index() + archive.readability_txt_file.do_stats() if archive.screenshot_file: archive.screenshot_file.associated = True - archive.screenshot_file.do_index() + archive.screenshot_file.do_stats() archive.singlefile_file.model = Archive.__tablename__ @@ -97,10 +96,6 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session): # Remove this group, it will not be processed by other modelers. del groups[stem] - if next(count) == 100: - # Commit occasionally. - session.commit() - @register_after_refresh @limit_concurrent(1) @@ -130,15 +125,14 @@ def archive_cleanup(): with get_db_session(commit=True) as session: # Remove any Domains without any Archives. domain_ids = [i[0] for i in session.execute('SELECT DISTINCT domain_id FROM archive') if i[0]] - deleted = 0 for domain in session.query(Domain): if domain.id not in domain_ids: - deleted += 1 session.delete(domain) @register_indexer('text/html') class ArchiveIndexer(Indexer, ABC): + """Gathers index data from the files associated with an Archive.""" @staticmethod def get_title(file: File): diff --git a/modules/archive/models.py b/modules/archive/models.py index 4270ac7c..3669e46e 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -12,6 +12,7 @@ from sqlalchemy.orm.collections import InstrumentedList from wrolpi.common import ModelHelper, Base, logger from wrolpi.dates import TZDateTime from wrolpi.errors import InvalidArchive +from wrolpi.files.lib import split_path_stem_and_suffix from wrolpi.files.models import File from wrolpi.media_path import MediaPathType @@ -78,6 +79,7 @@ class Archive(Base, ModelHelper): return self.archive_datetime > other.archive_datetime def __json__(self): + stem, _ = split_path_stem_and_suffix(self.singlefile_path) if self.singlefile_path else (None, None) d = dict( archive_datetime=self.archive_datetime, domain=self.domain.dict() if self.domain else None, @@ -88,7 +90,7 @@ class Archive(Base, ModelHelper): readability_txt_path=self.readability_txt_path, screenshot_path=self.screenshot_path, singlefile_path=self.singlefile_path, - stem=self.singlefile_path.stem if self.singlefile_path else None, + stem=stem, title=self.title, url=self.url, ) @@ -172,7 +174,8 @@ class Archive(Base, ModelHelper): readability_json_file = self.readability_json_file if not readability_json_file or not readability_json_file.path.is_file(): - raise InvalidArchive(f'Archive must have a readability json file: {self.singlefile_file.path}') + logger.warning(f'Archive must have a readability json file: {self.singlefile_file.path}') + return try: with readability_json_file.path.open() as fh: diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 666b729b..4390a2a9 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -12,6 +12,7 @@ from modules.archive.lib import get_or_create_domain, get_new_archive_files, del from modules.archive.models import Archive, Domain from wrolpi.dates import local_timezone from wrolpi.db import get_db_session +from wrolpi.files.lib import refresh_files from wrolpi.files.models import File from wrolpi.root_api import CustomJSONEncoder from wrolpi.test.common import skip_circleci @@ -430,6 +431,43 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_ assert response.json['files'][0]['archive']['archive_datetime'], 'Archive has no datetime' +@pytest.mark.asyncio +async def test_refresh_archives_index(test_session, test_directory, test_client, make_files_structure): + """Archives are indexed using ArchiveIndexer and archive_modeler.""" + # The start of a typical singlefile html file. + singlefile_contents = ''' ''' + + singlefile, *_ = make_files_structure({ + 'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents, + 'archive/example.com/2021-10-05-16-20-10_NA.png': None, + 'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents', + 'archive/example.com/2021-10-05-16-20-10_NA.readability.json': + '{"url": "https://example.com", "title": "the title"}', + 'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '', + }) + + await refresh_files() + + archive: Archive = test_session.query(Archive).one() + assert archive.singlefile_path == singlefile + assert archive.singlefile_file.a_text == 'the title' + assert archive.singlefile_file.d_text == 'article text contents' + + # The associated files are not indexed. + for name in ('screenshot_file', 'readability_file', 'readability_json_file', 'readability_txt_file'): + file = getattr(archive, name) + assert file.associated is True, 'The associated file is not marked' + assert file.indexed is False + # file.a_text is the file name. + assert not file.b_text + assert not file.c_text + assert not file.d_text + + def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure): """ Invalid Archives should be removed during refresh. diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index f22d008c..4feaa09a 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -1,4 +1,3 @@ -import itertools import json import pathlib from abc import ABC @@ -22,12 +21,10 @@ __all__ = ['video_modeler', 'VideoIndexer'] @register_modeler def video_modeler(groups: Dict[str, List[File]], session: Session): - count = itertools.cycle(range(101)) - - # Only process groups that contain a video file. local_groups = groups.copy() + videos = [] for stem, group in local_groups.items(): - video_file = next((i for i in group if i.mimetype.split('/')[0] == 'video' and i.path.suffix != '.part'), None) + video_file = next((i for i in group if i.mimetype.startswith('video/') and i.path.suffix != '.part'), None) if not video_file: # Not a video group. continue @@ -40,28 +37,34 @@ def video_modeler(groups: Dict[str, List[File]], session: Session): if poster_file: poster_file.associated = True - poster_file.do_index() + poster_file.do_stats() if caption_file: caption_file.associated = True - caption_file.do_index() + caption_file.do_stats() if info_json_file: info_json_file.associated = True - info_json_file.do_index() + info_json_file.do_stats() - video: Video = Video.upsert(video_file, session) + video = session.query(Video).filter_by(video_file=video_file).one_or_none() + if not video: + video = Video(video_file=video_file) + session.add(video) video.poster_file = poster_file video.caption_file = caption_file video.info_json_file = info_json_file + video.size = video_file.path.stat().st_size + video_file.model = Video.__tablename__ - video.video_file.do_index() - video.validate(session) + videos.append(video) - # Remove this group, it will not be processed by other modelers. - del groups[stem] + if videos: + session.flush(videos) + for video in videos: + video.video_file.do_index() + video.validate(session) - if next(count) == 100: - # Commit occasionally. - session.commit() + # Remove this group, it will not be processed by other modelers. + del groups[video.video_path.stem] @register_after_refresh diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py index baf69b04..f60ac4a6 100644 --- a/modules/videos/channel/test/test_lib.py +++ b/modules/videos/channel/test/test_lib.py @@ -7,6 +7,7 @@ from modules.videos.channel import lib from modules.videos.lib import save_channels_config, import_channels_config from modules.videos.models import Channel, Video from wrolpi.errors import UnknownChannel +from wrolpi.files.lib import refresh_files @pytest.mark.parametrize('params', [ @@ -93,7 +94,8 @@ def test_channels_no_url(test_session, test_directory): import_channels_config() -def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory): +@pytest.mark.asyncio +async def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory): """A Video will be assigned to a Channel by its info json first, then to its directory.""" channel1 = channel_factory(source_id='channel1') channel2 = channel_factory(source_id='channel2') @@ -110,20 +112,20 @@ def test_refresh_videos_finds_channel(test_session, test_client, channel_factory with mock.patch('wrolpi.common.after_refresh') as mock_after_refresh: # Channel can be found in `video_cleanup`, prevent that with mock. mock_after_refresh.side_effect = [] - test_client.post('/api/files/refresh') + await refresh_files() # Channel in info_json was found during validation. assert vid1.channel_id == channel1.id # Refresh using cleanup, info json should be trusted. - test_client.post('/api/files/refresh') + await refresh_files() assert vid1.channel_id == channel1.id # Remove the info json, the Video's Channel will be its directory. vid1.channel_id = None vid1.info_json_path.unlink() test_session.commit() - test_client.post('/api/files/refresh') + await refresh_files() test_session.refresh(vid1) vid1 = test_session.query(Video).one() assert vid1.channel_id == channel2.id diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 7cbebc31..9adbb6eb 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from wrolpi import before_startup -from wrolpi.common import ConfigFile, get_media_directory, sanitize_link +from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, after_refresh, register_after_refresh from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.files.models import File @@ -375,8 +375,14 @@ def save_channels_config(session=None, preserve_favorites: bool = True): @before_startup +@register_after_refresh def import_channels_config(): """Import channel settings to the DB. Existing channels will be updated.""" + if PYTEST and not TEST_CHANNELS_CONFIG: + logger.warning(f'Not importing channels during this test. Use `test_channels_config` fixture if you would' + f'like to call this.') + return + logger.info('Importing videos config') try: config = get_channels_config() diff --git a/modules/videos/models.py b/modules/videos/models.py index 38ad86ab..71f56dd2 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -13,7 +13,7 @@ from wrolpi.dates import now, TZDateTime from wrolpi.db import get_db_curs from wrolpi.downloader import Download, download_manager from wrolpi.errors import UnknownVideo -from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem +from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem, split_path_stem_and_suffix from wrolpi.files.models import File from wrolpi.media_path import MediaPathType @@ -240,7 +240,7 @@ class Video(ModelHelper, Base): def __json__(self): from modules.videos.common import minimize_video_info_json info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None - stem = self.video_path.stem if self.video_path else None + stem, _ = split_path_stem_and_suffix(self.video_path) if self.video_file else (None, None) d = dict( caption_path=self.caption_path, diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index ce76c5a5..4141556a 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -3,13 +3,41 @@ import shutil from http import HTTPStatus from pathlib import Path +import pytest + from modules.videos.models import Video, Channel from modules.videos.video import lib as video_lib from wrolpi.downloader import DownloadFrequency +from wrolpi.files.lib import refresh_files from wrolpi.files.models import File from wrolpi.vars import PROJECT_DIR +@pytest.mark.asyncio +async def test_refresh_videos_index(test_session, test_directory, video_factory): + """A video discovered during refresh has files associated with it. Only the video file is indexed.""" + video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True) + test_session.commit() + + test_session.query(Video).delete() + test_session.query(File).delete() + test_session.commit() + + for _ in range(1): + # Refreshing twice does not change the results. + await refresh_files() + + assert test_session.query(File).count() == 4 + video: Video = test_session.query(Video).one() + assert video.video_file.indexed is True, 'Video was not indexed' + assert video.video_file.a_text, 'Video title was not indexed' + assert video.video_file.d_text, 'Video captions were not indexed' + + assert not video.poster_file.indexed + assert not video.caption_file.indexed + assert not video.info_json_file.indexed + + def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory): subdir = test_directory / 'subdir' subdir.mkdir() diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 1009a999..6bcc7c85 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -70,14 +70,14 @@ def test_parse_video_file_name(file_name, expected): @pytest.mark.asyncio -async def test_associated_files(test_session, test_directory, video_file, image_file, vtt_file1): +async def test_associated_video_files(test_session, test_directory, video_file, image_file, vtt_file1): """A Video file has files related to it. They should be attached to the Video record.""" caption_file = video_file.with_suffix('.en.vtt') vtt_file1.rename(caption_file) poster_file = video_file.with_suffix('.jpeg') image_file.rename(poster_file) - await files_lib._refresh_all_files() + await files_lib.refresh_files() test_session.commit() # All files were found. @@ -93,7 +93,7 @@ async def test_associated_files(test_session, test_directory, video_file, image_ assert video.video_file.size assert video.video_file.indexed - await files_lib._refresh_all_files() + await files_lib.refresh_files() test_session.commit() # No new files were added. @@ -113,7 +113,7 @@ async def test_unassociated_video(test_session, test_directory, video_file): video_file.rename(new_video_file_path) video_file = new_video_file_path - await files_lib._refresh_all_files() + await files_lib.refresh_files() test_session.commit() video: Video = test_session.query(Video).one() diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 97bcc8f9..652a1662 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -34,8 +34,8 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic caption = video.video_file.d_text video = video.video_file.__json__() video['video']['caption'] = caption - previous_video = previous_video.video_file.__json__() if previous_video else None - next_video = next_video.video_file.__json__() if next_video else None + previous_video = previous_video.video_file.__json__() if previous_video and previous_video.video_file else None + next_video = next_video.video_file.__json__() if next_video and next_video.video_file else None return video, previous_video, next_video diff --git a/scripts/import_map.sh b/scripts/import_map.sh index 9863e348..716572f0 100755 --- a/scripts/import_map.sh +++ b/scripts/import_map.sh @@ -22,12 +22,12 @@ fi if [[ ${1} == *.osm.pbf ]]; then # Import a PBF file. - osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \ - /opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \ + nice -n 18 osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \ + /opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 3 \ -S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}" elif [[ ${1} == *.dump ]]; then # Import a Postgresql dump. - pg_restore -j4 --no-owner -d gis "${1}" + nice -n 18 pg_restore -j3 --no-owner -d gis "${1}" else echo "Cannot import unknown map file" exit 5 diff --git a/scripts/install_map_ubuntu_20.04.sh b/scripts/install_map_ubuntu_20.04.sh index 5ab52558..2040fc9e 100755 --- a/scripts/install_map_ubuntu_20.04.sh +++ b/scripts/install_map_ubuntu_20.04.sh @@ -18,6 +18,21 @@ apt install -y libboost-all-dev git tar unzip wget bzip2 build-essential autocon /usr/bin/npm install -g carto /usr/bin/carto -v +function yes_or_no { + while true; do + read -p "$* [y/n]:" yn + case $yn in + [Yy]*) return 0 ;; + [Nn]*) return 1 ;; + esac + done +} + +sudo -u postgres psql -c '\l' | grep gis && + yes_or_no "Map database already exists. Replace it?" && + systemctl stop renderd && + sudo -u postgres dropdb gis + # Create `gis` database for map. sudo -u postgres psql -c '\l' | grep gis || sudo -u postgres createdb -E UTF8 -O wrolpi gis sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" || : @@ -64,7 +79,7 @@ while [ "${COUNT}" -lt "${MAX_TRIES}" ]; do external_data=true [[ "${dc_import}" = false ]] && # Run import in "create" mode so basic tables will be created. - sudo -u wrolpi osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \ + sudo -u wrolpi nice -n 18 osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \ /opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \ -S /opt/openstreetmap-carto/openstreetmap-carto.style /tmp/district-of-columbia-latest.osm.pbf && dc_import=true diff --git a/wrolpi/common.py b/wrolpi/common.py index 9b3a1de8..26b1e734 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1,4 +1,5 @@ import asyncio +import atexit import contextlib import inspect import logging @@ -26,7 +27,7 @@ from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session -from wrolpi.dates import DEFAULT_TIMEZONE +from wrolpi.dates import DEFAULT_TIMEZONE, now from wrolpi.errors import WROLModeEnabled, NativeOnly from wrolpi.vars import PYTEST, MODULES_DIR, \ DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY @@ -37,6 +38,58 @@ formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(messag ch.setFormatter(formatter) logger.addHandler(ch) +__all__ = [ + 'Base', + 'ModelHelper', + 'get_model_by_table_name', + 'tsvector', + 'compile_tsvector', + 'sanitize_link', + 'ConfigFile', + 'get_config', + 'set_test_config', + 'wrol_mode_enabled', + 'wrol_mode_check', + 'enable_wrol_mode', + 'disable_wrol_mode', + 'insert_parameter', + 'iterify', + 'date_range', + 'remove_whitespace', + 'run_after', + 'set_test_media_directory', + 'get_media_directory', + 'check_media_directory', + 'get_absolute_media_path', + 'get_relative_to_media_directory', + 'get_files_and_directories', + 'minimize_dict', + 'make_media_directory', + 'extract_domain', + 'import_modules', + 'api_param_limiter', + 'partition', + 'chdir', + 'zig_zag', + 'walk', + 'escape_file_name', + 'native_only', + 'recursive_map', + 'any_extensions', + 'aiohttp_post', + 'register_modeler', + 'apply_modelers', + 'register_after_refresh', + 'apply_after_refresh', + 'match_paths_to_suffixes', + 'chunks', + 'chunks_by_name', + 'timer', + 'cum_timer', + 'limit_concurrent', + 'truncate_object_bytes' +] + # Base is used for all SQLAlchemy models. Base = declarative_base() @@ -369,11 +422,6 @@ def date_range(start: dt_or_d, end: dt_or_d, steps: int) -> List[dt_or_d]: return [start + (delta * i) for i in range(steps)] -def chunks(it, size): - it = iter(it) - return iter(lambda: tuple(islice(it, size)), ()) - - WHITESPACE = re.compile(r'\s') @@ -763,7 +811,7 @@ def apply_modelers(files, session: Session): for file in files: if not file.mimetype: file.mimetype = get_mimetype(file.path) - stem, suffix = split_path_stem_and_suffix(file.path) + stem, _ = split_path_stem_and_suffix(file.path) try: groups[stem].append(file) except KeyError: @@ -771,7 +819,6 @@ def apply_modelers(files, session: Session): for modeler in modelers: modeler(groups, session) - session.commit() for stem, group in groups.items(): # Index any files that were not claimed by modelers. @@ -784,6 +831,7 @@ after_refresh = [] def register_after_refresh(func: callable): after_refresh.append(func) + return func def apply_after_refresh(): @@ -807,6 +855,56 @@ def match_paths_to_suffixes(paths: List[pathlib.Path], suffix_groups: List[Tuple yield match +def chunks(it: Iterable, size: int): + """Split an iterable into iterables of the defined length.""" + it = iter(it) + return iter(lambda: tuple(islice(it, size)), ()) + + +def chunks_by_name(it: List[Union[pathlib.Path, str]], size: int) -> Generator[List[pathlib.Path], None, None]: + """ + Attempt to split a list of paths near the defined size. Keep groups of files together when they share + matching names. + + >>> files = ['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4'] + >>> chunks_by_name(files, 2) + [['1.mp4', '1.txt'], ['2.mp4', '2.txt', '2.png'], ['3.mp4']] + >>> chunks_by_name(files, 3) + [['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png'], ['3.mp4']] + """ + if not isinstance(size, int) or size < 1: + raise ValueError('size must be a positive integer') + + if not it or len(it) < size: + yield it + return + + if not isinstance(it, list): + raise ValueError('iterable must a list') + + from wrolpi.files.lib import split_path_stem_and_suffix + it = sorted(it.copy()) + index = size + last_name = None + while it: + if index >= len(it): + # Ran out of items, yield what is left. + yield it + return + path = it[index] + name, _ = split_path_stem_and_suffix(path) + if last_name and name != last_name: + # Found a break in the path names, yield and reset. + chunk, it = it[:index], it[index:] + index = size + last_name = None + yield chunk + continue + # Didn't find a name change, try again. + last_name = name + index += 1 + + @contextlib.contextmanager def timer(name): """Prints out the time elapsed during the call of some block.""" @@ -817,6 +915,35 @@ def timer(name): logger.warning(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds') +TIMERS = dict() + + +@contextlib.contextmanager +def cum_timer(name: str): + """Track time usage within this context's name. The cumulative calls to this context will be printed on exit.""" + before = now() + try: + yield + finally: + total_seconds = (now() - before).total_seconds() + existing_seconds, calls = TIMERS.get(name, (0, 0)) + TIMERS[name] = (existing_seconds + total_seconds, calls + 1) + + +@atexit.register +def print_timer(): + """Print any cumulative timers that have been stored during exit.""" + if not TIMERS: + return + + total = sum(i[0] for i in TIMERS.values()) + for name, (seconds, calls) in sorted(TIMERS.items(), key=lambda i: i[1]): + percent = int((seconds / total) * 100) + seconds = round(seconds, 5) + print(f'CUM_TIMER: {repr(name)} elapsed {seconds} cumulative seconds ({percent}%)' + f' {calls} calls', file=sys.stderr) + + def limit_concurrent(limit: int): """ Wrapper that limits the amount of concurrently running functions. @@ -852,10 +979,6 @@ def limit_concurrent(limit: int): return wrapper -async def count_files(path: pathlib.Path) -> int: - return len([i for i in walk(path) if i.is_file()]) - - def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> List[str]: """ Shorten an object. This is useful when inserting something into a tsvector. diff --git a/wrolpi/db.py b/wrolpi/db.py index d7233268..2259839c 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -1,3 +1,4 @@ +import logging from contextlib import contextmanager from functools import wraps from typing import ContextManager, Tuple, List, Union, Type @@ -41,7 +42,7 @@ postgres_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{d # This engine is used for all normal tasks (except testing). db_args = get_db_args() -connect_args = dict(application_name='wrolpi_api') +connect_args = dict(application_name='wrolpi_api', connect_timeout=600) uri = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**db_args) engine = create_engine(uri, poolclass=NullPool, connect_args=connect_args) session_maker = sessionmaker(bind=engine) @@ -75,6 +76,9 @@ def get_db_context() -> Tuple[create_engine, Session]: return local_engine, session +# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + @contextmanager def get_db_session(commit: bool = False) -> ContextManager[Session]: """ diff --git a/wrolpi/files/indexers.py b/wrolpi/files/indexers.py index 35695823..609840c7 100644 --- a/wrolpi/files/indexers.py +++ b/wrolpi/files/indexers.py @@ -103,6 +103,8 @@ class TextIndexer(Indexer, ABC): Detects VTT (caption) files and forwards them on.""" + _ignored_suffixes = ('.json', '.vtt', '.srt', '.csv') + @classmethod def create_index(cls, file): path = file.path.path if hasattr(file.path, 'path') else file.path @@ -120,10 +122,7 @@ class TextIndexer(Indexer, ABC): @classmethod def detect_special_indexer(cls, file): - if file.path.name.endswith('.json'): - # JSON files are detected as "text/plain" but we don't want them to be indexed. - return DefaultIndexer - if file.path.name.endswith('.vtt') or file.path.name.endswith('.srt'): - # Don't process caption files. + if any(file.path.name.endswith(i) for i in cls._ignored_suffixes): + # Some text files aren't for humans. return DefaultIndexer return cls diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 7b36bb69..00a72194 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -4,6 +4,7 @@ import glob import pathlib import re import subprocess +from asyncio import Task from functools import wraps from pathlib import Path from typing import List, Tuple, Union @@ -13,12 +14,12 @@ from sqlalchemy.orm import Session from wrolpi.cmd import which from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \ - get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name + get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name, chunks_by_name from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidFile from wrolpi.files.models import File -from wrolpi.vars import PYTEST +from wrolpi.vars import PYTEST, FILE_REFRESH_CHUNK_SIZE try: import magic @@ -88,20 +89,6 @@ def delete_file(file: str): FILE_NAME_REGEX = re.compile(r'[_ .]') - -def split_file_name_into_words(path: Path) -> List[str]: - """ - Split a file name into words. - - >>> split_path_stem_and_suffix(Path('foo.mp4')) - ['foo'] - >>> split_path_stem_and_suffix(Path('foo bar.mp4')) - ['foo', 'bar'] - """ - words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)] - return words - - FILE_BIN = which('file', '/usr/bin/file') @@ -144,55 +131,88 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str]) -> Tuple[str, str return path.name, '' +refresh_logger = logger.getChild('refresh') + + @wrol_mode_check -def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime): +async def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime): """Upsert, index, and model all provided paths.""" # Update the idempotency if any files already in the DB. if not paths: return - paths = set(paths) + for idx, chunk in enumerate(map(set, chunks_by_name(paths, FILE_REFRESH_CHUNK_SIZE))): + with get_db_session(commit=True) as session: + existing_files = list(session.query(File).filter(File.path.in_(chunk))) + existing_paths = {i.path for i in existing_files} + # Add any new files into the database. + new_files = [] + if new_paths := (chunk - existing_paths): + new_files = [File(path=i, idempotency=idempotency, mimetype=get_mimetype(i)) for i in new_paths] + session.add_all(new_files) + session.flush(new_files) + # Apply models to all files. Many files will have no model. + all_files = existing_files + new_files + apply_modelers(all_files, session) - with get_db_session(commit=True) as session: - existing_files = list(session.query(File).filter(File.path.in_(paths))) - for existing_file in existing_files: - existing_file.idempotency = idempotency - session.commit() + # Update idempotency for existing files after modelers. Otherwise SQLAlchemy forgets. + for existing_file in existing_files: + existing_file.idempotency = idempotency - existing_paths = {i.path for i in existing_files} - # Add any new files into the database. - new_files = [] - if new_paths := (paths - existing_paths): - new_files = [File(path=i, idempotency=idempotency) for i in new_paths] - session.add_all(new_files) - # Commit before modeling. Just in case modeling fails. - session.commit() + if idx > 0: + parent = all_files[0].path.parent + refresh_logger.debug(f'Committed chunk of {len(chunk)} files in {parent}') - # Apply models to all files. Many files will have no model. - apply_modelers(existing_files + new_files, session) + # Sleep between chunks to catch cancel. + await asyncio.sleep(0) -refresh_logger = logger.getChild('refresh') - - -async def _refresh_directory_files_recursively(directory, idempotency): - """ - Refresh all files in a directory, recursively refresh all files in subdirectories. - """ +async def _refresh_directory_files_recursively(directory: pathlib.Path, idempotency: datetime): + """Refresh all files in a directory, recursively refresh all files in subdirectories.""" directories = [directory, ] while directories: directory = directories.pop(0) try: files, new_directories = get_files_and_directories(directory) - refresh_logger.debug(f'Refreshing {len(files)} files in {directory}') directories.extend(new_directories) - _refresh_files_list(files, idempotency) + if files: + refresh_logger.info(f'Refreshing {len(files)} files in {directory}') + await _refresh_files_list(files, idempotency) except Exception as e: - logger.error(f'Failed to refresh files in {directory}', exc_info=e) + refresh_logger.error(f'Failed to refresh files in {directory}', exc_info=e) + except asyncio.CancelledError: + refresh_logger.error(f'Refresh canceled during {directory}') + raise + + +REFRESH_TASKS: List[Task] = [] + + +async def cancel_refresh_tasks(): + """Cancel all refresh tasks, if any.""" + if REFRESH_TASKS: + refresh_logger.warning(f'Canceling {len(REFRESH_TASKS)} refreshes') + for task in REFRESH_TASKS: + task.cancel() + await asyncio.gather(*REFRESH_TASKS) + + +def cancelable_wrapper(func: callable): + @wraps(func) + async def wrapped(*args, **kwargs): + if PYTEST: + return await func(*args, **kwargs) + + task = asyncio.create_task(func(*args, **kwargs)) + REFRESH_TASKS.append(task) + + return wrapped @limit_concurrent(1) # Only one refresh at a time. -async def _refresh_all_files(): +@wrol_mode_check +@cancelable_wrapper +async def refresh_files(): """Find, model, and index all files in the media directory.""" refresh_logger.warning('Refreshing all files') @@ -217,18 +237,9 @@ async def _refresh_all_files(): refresh_logger.warning('Done refreshing Files') -@wraps(_refresh_all_files) -@wrol_mode_check -async def refresh_files(): - """Schedule a refresh task if not testing. If testing, do a synchronous refresh.""" - if PYTEST: - return await _refresh_all_files() - - asyncio.create_task(_refresh_all_files()) - - @limit_concurrent(1) @wrol_mode_check +@cancelable_wrapper async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str]): """Upsert and index all files within a directory (recursively). diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 812bfbe0..1612e2c4 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -91,18 +91,7 @@ class File(ModelHelper, Base): def do_index(self, force_index: bool = False): """Gather any missing information about this file. Index the contents of this file using an Indexer.""" - from wrolpi.files.lib import split_path_stem_and_suffix - - if not self.mimetype: - from wrolpi.files.lib import get_mimetype - self.mimetype = get_mimetype(self.path) - - self.title = self.title or self.path.name - _, self.suffix = split_path_stem_and_suffix(self.path) - - stat = self.path.stat() - self.size = stat.st_size - self.modification_datetime = from_timestamp(stat.st_mtime) + self.do_stats() try: if force_index or self.indexed is not True: @@ -119,6 +108,22 @@ class File(ModelHelper, Base): except Exception as e: logger.error(f'Failed to index {self.path}', exc_info=e) + def do_stats(self): + """Assign the mimetype, title, size, modification_time of this file.""" + from wrolpi.files.lib import split_path_stem_and_suffix + if not self.mimetype: + from wrolpi.files.lib import get_mimetype + self.mimetype = get_mimetype(self.path) + self.title = self.title or self.path.name + stat = self.path.stat() + self.size = stat.st_size + self.modification_datetime = from_timestamp(stat.st_mtime) + name, self.suffix = split_path_stem_and_suffix(self.path) + + # Use title as the a_text temporarily, this may be overwritten by indexer. + # TODO this is very slow. What can be done instead? + self.a_text = self.a_text or name + @validates('path') def validate_path(self, key, value): value = pathlib.Path(value) if not isinstance(value, pathlib.Path) else value diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 9465d7ca..db84e49e 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1,8 +1,11 @@ +import asyncio import json import shutil import zipfile +from datetime import datetime from pathlib import Path from typing import List +from uuid import uuid4 import mock import pytest @@ -140,21 +143,6 @@ def test_delete_file(make_files_structure, test_directory): assert (test_directory / 'baz').is_dir() -@pytest.mark.parametrize( - 'path,expected', - [ - ('foo.mp4', ['foo']), - ('foo bar.txt', ['foo', 'bar']), - ('foo_bar.mp4', ['foo', 'bar']), - ('foo-bar', ['foo-bar']), - ('123 foo bar', ['123', 'foo', 'bar']), - ('123foo bar', ['123foo', 'bar']), - ] -) -def test_split_file_name_into_words(path, expected): - assert lib.split_file_name_into_words(Path(path)) == expected - - @pytest.mark.parametrize( 'path,expected', [ @@ -197,6 +185,42 @@ async def test_refresh_files(test_session, make_files_structure, test_directory) assert_files(test_session, []) +@pytest.mark.asyncio +async def test_refresh_many_files(test_session, make_files_structure): + count = 1_000 + make_files_structure([f'{uuid4()}.txt' for _ in range(count)]) + await lib.refresh_files() + assert test_session.query(File).count() == count + + await lib.refresh_files() + assert test_session.query(File).count() == count + + +@pytest.mark.asyncio +async def test_refresh_cancel(test_session, make_files_structure, test_directory): + """Refresh tasks can be canceled.""" + # Creat a lot of files so the refresh will take too long. + make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)]) + + async def assert_cancel(task_): + # Time the time it takes to cancel. + before = datetime.now() + # Sleep so the refresh task has time to run. + await asyncio.sleep(0.1) + + # Cancel the refresh (it will be sleeping soon). + task_.cancel() + with pytest.raises(asyncio.CancelledError): + await task_ + assert (datetime.now() - before).total_seconds() < 0.5, 'Task took too long. Was the refresh canceled?' + + task = asyncio.create_task(lib.refresh_files()) + await assert_cancel(task) + + task = asyncio.create_task(lib.refresh_directory_files_recursively(test_directory)) + await assert_cancel(task) + + @pytest.mark.asyncio async def test_refresh_files_in_directory(test_session, make_files_structure, test_directory): """A subdirectory can be refreshed, files above it can be ignored.""" @@ -276,7 +300,7 @@ async def test_files_indexer(test_session, make_files_structure): # Enable slow feature for testing. # TODO can this be sped up to always be included? with mock.patch('modules.videos.EXTRACT_SUBTITLES', True): - await lib._refresh_all_files() + await lib.refresh_files() bzip_file, gzip_file, text_file, zip_file, image_file, unknown_file, info_json_file, video_file \ = test_session.query(File).order_by(File.path) @@ -330,7 +354,7 @@ async def test_large_text_indexer(test_session, make_files_structure): large, = make_files_structure({ 'large_file.txt': 'foo ' * 1_000_000, }) - await lib._refresh_all_files() + await lib.refresh_files() assert test_session.query(File).count() == 1 assert large.is_file() and large.stat().st_size == 4_000_000 diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index d4343ba7..4c67b69b 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -1,5 +1,5 @@ import json -import multiprocessing +import json import re from datetime import datetime, date from decimal import Decimal @@ -24,14 +24,14 @@ from wrolpi.dates import set_timezone from wrolpi.downloader import download_manager from wrolpi.errors import WROLModeEnabled, InvalidTimezone, API_ERRORS, APIError, ValidationError, HotspotError from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse +from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \ + truthy_arg from wrolpi.version import __version__ logger = logger.getChild(__name__) api_app = Sanic(name='api_app') -DEFAULT_HOST, DEFAULT_PORT = '127.0.0.1', '8081' - api_bp = Blueprint('RootAPI', url_prefix='/api') BLUEPRINTS = [api_bp, ] @@ -50,26 +50,47 @@ def add_blueprint(bp: Union[Blueprint, BlueprintGroup]): BLUEPRINTS.append(bp) -def run_webserver(host: str, port: int, workers: int = 8): +def run_webserver( + host: str = API_HOST, + port: int = API_PORT, + workers: int = API_WORKERS, + api_debug: bool = API_DEBUG, + access_log: bool = API_ACCESS_LOG, +): # Attach all blueprints after they have been defined. for bp in BLUEPRINTS: api_app.blueprint(bp) - kwargs = dict(host=host, port=port, workers=workers, debug=False, access_log=False, auto_reload=True) - logger.debug(f'Running Sanic {sanic_version} with kwargs {kwargs}') + kwargs = dict( + host=host, + port=port, + workers=workers, + debug=api_debug, + access_log=access_log, + auto_reload=DOCKERIZED, + ) + logger.warning(f'Running Sanic {sanic_version} with kwargs {kwargs}') return api_app.run(**kwargs) def init_parser(parser): # Called by WROLPI's main() function - parser.add_argument('-H', '--host', default=DEFAULT_HOST, help='What network interface to connect webserver') - parser.add_argument('-p', '--port', default=DEFAULT_PORT, type=int, help='What port to connect webserver') - parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(), type=int, - help='How many web workers to run') + parser.add_argument('-H', '--host', default=API_HOST, help='What network interface to connect webserver') + parser.add_argument('-p', '--port', default=API_PORT, type=int, help='What port to connect webserver') + parser.add_argument('-w', '--workers', default=API_WORKERS, type=int, help='How many web workers to run') + parser.add_argument('--access-log', default=API_ACCESS_LOG, type=truthy_arg, help='Enable Sanic access log') + parser.add_argument('--api-debug', default=API_DEBUG, type=truthy_arg, help='Enable Sanic debug log') + parser.add_argument('--api-auto-reload', default=API_AUTO_RELOAD, type=truthy_arg, help='Enable Sanic auto reload') def main(args): - return run_webserver(args.host, args.port, args.workers) + return run_webserver( + host=args.host, + port=args.port, + workers=args.workers, + api_debug=args.api_debug, + access_log=args.access_log, + ) index_html = ''' @@ -313,14 +334,21 @@ async def throttle_off(_: Request): @openapi.description('Get the status of CPU/load/etc.') async def get_status(_: Request): s = await status.get_status() - downloads = download_manager.get_summary() + try: + downloads = download_manager.get_summary() + except Exception as e: + logger.error('Unable to get download status', exc_info=e) + downloads = dict() + ret = dict( bandwidth=s.bandwidth, cpu_info=s.cpu_info, + disk_bandwidth=s.disk_bandwidth, downloads=downloads, drives=s.drives, hotspot_status=admin.hotspot_status().name, load=s.load, + memory_stats=s.memory_stats, throttle_status=admin.throttle_status().name, version=__version__, wrol_mode=wrol_mode_enabled(), diff --git a/wrolpi/status.py b/wrolpi/status.py index 490eadef..6676de3e 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import asyncio import multiprocessing +import pathlib import re import subprocess from dataclasses import dataclass @@ -9,7 +10,7 @@ from pathlib import Path from typing import List, Optional, Tuple from wrolpi.cmd import which -from wrolpi.common import logger +from wrolpi.common import logger, limit_concurrent from wrolpi.dates import now try: @@ -41,9 +42,9 @@ class SystemLoad: def __json__(self): return dict( - minute_1=str(self.minute_1), - minute_5=str(self.minute_5), - minute_15=str(self.minute_15), + minute_1=str(round(self.minute_1, 2)), + minute_5=str(round(self.minute_5, 2)), + minute_15=str(round(self.minute_15, 2)), ) @@ -179,6 +180,60 @@ async def get_cpu_info() -> CPUInfo: return info +MEMORY_TOTAL = re.compile(r'^MemTotal:\s+(\d+)\s+kB$', re.MULTILINE) +MEMORY_FREE = re.compile(r'^MemFree:\s+(\d+)\s+kB$', re.MULTILINE) +MEMORY_CACHED = re.compile(r'^Cached:\s+(\d+)\s+kB$', re.MULTILINE) + + +@dataclass +class MemoryStats: + total: int = None + used: int = None + free: int = None + cached: int = None + + def __json__(self): + return dict( + total=self.total, + used=self.used, + free=self.free, + cached=self.cached, + ) + + +def get_memory_stats_psutil(): + mem = psutil.virtual_memory() + mem = MemoryStats( + total=mem.total, + used=mem.used, + free=mem.free, + cached=mem.cached, + ) + return mem + + +async def get_memory_stats(): + try: + return get_memory_stats_psutil() + except Exception as e: + warn_once(e) + + meminfo = pathlib.Path('/proc/meminfo') + if meminfo.is_file() and (contents := meminfo.read_text()): + total = int(i[0]) if (i := MEMORY_TOTAL.findall(contents)) else None + free = int(i[0]) if (i := MEMORY_FREE.findall(contents)) else None + cached = int(i[0]) if (i := MEMORY_CACHED.findall(contents)) else None + mem = MemoryStats( + total=int(total * 1024) if total else None, + free=int(free * 1024) if free else None, + cached=int(cached * 1024) if cached else None, + used=int(total - free) if total and free else None, + ) + return mem + + return MemoryStats() + + @dataclass class DriveInfo: mount: str = None @@ -263,7 +318,7 @@ async def get_drives_info() -> List[DriveInfo]: @dataclass -class BandwidthInfo: +class NICBandwidthInfo: bytes_recv: int = None bytes_sent: int = None elapsed: int = None @@ -288,6 +343,8 @@ IGNORED_NIC_NAMES = { 'br-', } +BANDWIDTH = multiprocessing.Manager().dict() + def get_nic_names() -> List[str]: """Finds all non-virtual and non-docker network interface names.""" @@ -299,31 +356,57 @@ def get_nic_names() -> List[str]: return names -BANDWIDTH = multiprocessing.Manager().dict() +@dataclass +class DiskBandwidthInfo: + bytes_read: int = None + bytes_write: int = None + elapsed: int = None + name: str = None + + def __json__(self): + return dict( + bytes_read=self.bytes_read, + bytes_write=self.bytes_write, + elapsed=self.elapsed, + name=self.name, + ) -async def get_bandwidth_info() -> List[BandwidthInfo]: - """ - Get all bandwidth information for all NICs. - """ - infos = [] +DRIVE_BANDWIDTH = multiprocessing.Manager().dict() + + +async def get_bandwidth_info() -> Tuple[List[NICBandwidthInfo], List[DiskBandwidthInfo]]: + """Get all bandwidth information for all NICs.""" + nics_info = [] + disks_info = [] try: for name in sorted(BANDWIDTH.keys()): nic = BANDWIDTH[name] if 'bytes_recv_ps' not in nic: # Not stats collected yet. continue - infos.append(BandwidthInfo( + nics_info.append(NICBandwidthInfo( bytes_recv=nic['bytes_recv_ps'], bytes_sent=nic['bytes_sent_ps'], elapsed=nic['elapsed'], name=name, speed=nic['speed'], )) + for name in sorted(DRIVE_BANDWIDTH.keys()): + disk = DRIVE_BANDWIDTH[name] + if 'bytes_read_ps' not in disk: + # No status collected yet + continue + disks_info.append(DiskBandwidthInfo( + bytes_read=disk['bytes_read_ps'], + bytes_write=disk['bytes_write_ps'], + elapsed=disk['elapsed'], + name=name, + )) except Exception as e: warn_once(e) - return infos + return nics_info, disks_info def _get_nic_tick(name_): @@ -341,7 +424,7 @@ def _get_nic_tick(name_): def _calculate_bytes_per_second(history: List[Tuple]) -> Tuple[int, int, int]: """Calculate the bytes-per-second between the oldest and newest tick.""" - (oldest_now, oldest_recv, oldest_sent, _), (newest_now, newest_recv, newest_sent, _) = \ + (oldest_now, oldest_recv, oldest_sent, *_), (newest_now, newest_recv, newest_sent, *_) = \ history[0], history[-1] elapsed = int(newest_now - oldest_now) if elapsed == 0: @@ -351,6 +434,7 @@ def _calculate_bytes_per_second(history: List[Tuple]) -> Tuple[int, int, int]: return bytes_recv_ps, bytes_sent_ps, elapsed +@limit_concurrent(1) async def bandwidth_worker(count: int = None): """A background process which will gather historical data about all NIC bandwidth statistics.""" if not psutil: @@ -371,6 +455,17 @@ async def bandwidth_worker(count: int = None): nic['historical'] = (nic['historical'] + [_get_nic_tick(name_), ])[-21:] BANDWIDTH.update({name_: nic}) + timestamp = now().timestamp() + for name_, disk in psutil.disk_io_counters(perdisk=True).items(): + tic = timestamp, disk.read_bytes, disk.write_bytes + if bw := DRIVE_BANDWIDTH.get(name_): + bw['historical'] = (bw['historical'] + [tic, ])[-21:] + DRIVE_BANDWIDTH.update({name_: bw}) + else: + DRIVE_BANDWIDTH.update({ + name_: dict(historical=[tic, ]), + }) + # Initialize the stats. append_all_stats() @@ -394,6 +489,20 @@ async def bandwidth_worker(count: int = None): speed=historical[-1][-1], # Use the most recent speed. ) }) + for name, stats in DRIVE_BANDWIDTH.items(): + if 'historical' not in stats: + continue + + historical = stats['historical'] + bytes_read_ps, bytes_write_ps, elapsed = _calculate_bytes_per_second(historical) + DRIVE_BANDWIDTH.update({ + name: dict( + historical=historical, + bytes_read_ps=bytes_read_ps, + bytes_write_ps=bytes_write_ps, + elapsed=elapsed, + ) + }) @dataclass @@ -401,21 +510,26 @@ class Status: cpu_info: CPUInfo load: SystemLoad drives: List[DriveInfo] - bandwidth: BandwidthInfo + bandwidth: NICBandwidthInfo + disk_bandwidth: DiskBandwidthInfo + memory_stats: MemoryStats async def get_status() -> Status: - cpu_info, load, drives, bandwidth = await asyncio.gather( + cpu_info, load, drives, memory_stats, (nic_bandwidth, disk_bandwidth) = await asyncio.gather( get_cpu_info(), get_load(), get_drives_info(), + get_memory_stats(), get_bandwidth_info(), ) return Status( cpu_info=cpu_info, load=load, drives=drives, - bandwidth=bandwidth, + bandwidth=nic_bandwidth, + disk_bandwidth=disk_bandwidth, + memory_stats=memory_stats, ) @@ -428,6 +542,7 @@ if __name__ == '__main__': task = asyncio.gather( get_cpu_info(), + get_memory_stats(), get_load(), get_drives_info(), get_bandwidth_info(), diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 73fd1428..8d64f152 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -4,12 +4,13 @@ import tempfile import unittest from datetime import date, datetime from decimal import Decimal +from itertools import zip_longest from unittest import mock import pytest -from wrolpi.common import insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \ - escape_file_name, match_paths_to_suffixes, truncate_object_bytes, check_media_directory +import wrolpi.vars +from wrolpi import common from wrolpi.dates import set_timezone, now from wrolpi.errors import InvalidTimezone from wrolpi.test.common import build_test_directories @@ -63,51 +64,51 @@ def test_insert_parameter(): def func(foo, bar): pass - results = insert_parameter(func, 'bar', 'bar', (1,), {}) + results = common.insert_parameter(func, 'bar', 'bar', (1,), {}) assert results == ((1, 'bar'), {}) def func(foo, bar, baz): pass - results = insert_parameter(func, 'bar', 'bar', (1, 2), {}) + results = common.insert_parameter(func, 'bar', 'bar', (1, 2), {}) assert results == ((1, 'bar', 2), {}) def func(foo, baz, bar=None): pass - results = insert_parameter(func, 'bar', 'bar', (1, 2), {}) + results = common.insert_parameter(func, 'bar', 'bar', (1, 2), {}) assert results == ((1, 2, 'bar'), {}) def func(foo, baz, bar=None): pass - results = insert_parameter(func, 'baz', 'baz', (1, 2), {}) + results = common.insert_parameter(func, 'baz', 'baz', (1, 2), {}) assert results == ((1, 'baz', 2), {}) def func(foo, baz, qux=None, bar=None): pass - results = insert_parameter(func, 'bar', 'bar', (1, 2, 3), {}) + results = common.insert_parameter(func, 'bar', 'bar', (1, 2, 3), {}) assert results == ((1, 2, 3, 'bar'), {}) # bar is not defined as a parameter! def func(foo): pass - pytest.raises(TypeError, insert_parameter, func, 'bar', 'bar', (1,), {}) + pytest.raises(TypeError, common.insert_parameter, func, 'bar', 'bar', (1,), {}) class TestCommon(unittest.TestCase): def test_date_range(self): # A single step results in the start. - result = date_range(date(1970, 1, 1), date(1970, 1, 2), 1) + result = common.date_range(date(1970, 1, 1), date(1970, 1, 2), 1) assert result == [ date(1970, 1, 1), ] # Many steps on a single day results in the same day. - result = date_range(date(1970, 1, 1), date(1970, 1, 1), 5) + result = common.date_range(date(1970, 1, 1), date(1970, 1, 1), 5) assert result == [ date(1970, 1, 1), date(1970, 1, 1), @@ -117,7 +118,7 @@ class TestCommon(unittest.TestCase): ] # Many steps on a single datetime results in a range of times. - result = date_range(datetime(1970, 1, 1), datetime(1970, 1, 1, 23, 59, 59), 5) + result = common.date_range(datetime(1970, 1, 1), datetime(1970, 1, 1, 23, 59, 59), 5) assert result == [ datetime(1970, 1, 1, 0, 0), datetime(1970, 1, 1, 4, 47, 59, 800000), @@ -126,8 +127,8 @@ class TestCommon(unittest.TestCase): datetime(1970, 1, 1, 19, 11, 59, 200000), ] - # date_range is not inclusive, like range(). - result = date_range(date(1970, 1, 1), date(1970, 1, 5), 4) + # common.date_range is not inclusive, like range(). + result = common.date_range(date(1970, 1, 1), date(1970, 1, 5), 4) assert result == [ date(1970, 1, 1), date(1970, 1, 2), @@ -136,7 +137,7 @@ class TestCommon(unittest.TestCase): ] # Reversed dates are supported. - result = date_range(date(1970, 1, 5), date(1970, 1, 1), 4) + result = common.date_range(date(1970, 1, 5), date(1970, 1, 1), 4) assert result == [ date(1970, 1, 5), date(1970, 1, 4), @@ -145,7 +146,7 @@ class TestCommon(unittest.TestCase): ] # Large date spans are supported. - result = date_range(date(1970, 1, 1), date(2020, 5, 1), 4) + result = common.date_range(date(1970, 1, 1), date(2020, 5, 1), 4) assert result == [ date(1970, 1, 1), date(1982, 8, 1), @@ -153,7 +154,7 @@ class TestCommon(unittest.TestCase): date(2007, 10, 1), ] - result = date_range(datetime(1970, 1, 1, 0, 0, 0), datetime(1970, 1, 1, 10, 0), 8) + result = common.date_range(datetime(1970, 1, 1, 0, 0, 0), datetime(1970, 1, 1, 10, 0), 8) assert result == [ datetime(1970, 1, 1, 0, 0), datetime(1970, 1, 1, 1, 15), @@ -166,7 +167,7 @@ class TestCommon(unittest.TestCase): ] # More steps than days - result = date_range(date(1970, 1, 1), date(1970, 1, 7), 10) + result = common.date_range(date(1970, 1, 1), date(1970, 1, 7), 10) assert result == [ date(1970, 1, 1), date(1970, 1, 1), @@ -215,7 +216,7 @@ class TestCommon(unittest.TestCase): ] ) def test_api_param_limiter(i, expected): - limiter = api_param_limiter(100) # should never return an integer greater than 100. + limiter = common.api_param_limiter(100) # should never return an integer greater than 100. assert limiter(i) == expected @@ -227,22 +228,22 @@ def test_chdir(): home = os.environ.get('HOME') assert home - with chdir(): + with common.chdir(): assert os.getcwd() != original assert str(os.getcwd()).startswith('/tmp') assert os.environ['HOME'] != os.getcwd() # Replace $HOME - with chdir(with_home=True): + with common.chdir(with_home=True): assert os.getcwd() != original assert str(os.getcwd()).startswith('/tmp') assert os.environ['HOME'] == os.getcwd() with tempfile.TemporaryDirectory() as d: # Without replacing $HOME - with chdir(pathlib.Path(d), with_home=True): + with common.chdir(pathlib.Path(d), with_home=True): assert os.getcwd() == d assert os.environ['HOME'] == os.getcwd() - with chdir(pathlib.Path(d)): + with common.chdir(pathlib.Path(d)): assert os.getcwd() == d assert os.environ['HOME'] != os.getcwd() @@ -271,7 +272,7 @@ def test_chdir(): ]) ]) def test_zig_zag(low, high, expected): - zagger = zig_zag(low, high) + zagger = common.zig_zag(low, high) for i in expected: result = next(zagger) assert result == i @@ -297,7 +298,7 @@ def test_zig_zag(low, high, expected): ] ) def test_escape_file_name(name, expected): - assert escape_file_name(name) == expected + assert common.escape_file_name(name) == expected @pytest.mark.parametrize( @@ -319,37 +320,121 @@ def test_escape_file_name(name, expected): def test_match_paths_to_suffixes(paths, suffix_groups, expected): paths = [pathlib.Path(i) for i in paths] expected = tuple(pathlib.Path(i) if i else None for i in expected) - assert (i := match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}' + assert (i := common.match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}' def test_truncate_object_bytes(): """ Objects can be truncated (lists will be shortened) so they will fit in tsvector columns. """ - assert truncate_object_bytes(['foo'] * 10, 100) == ['foo'] * 5 - assert truncate_object_bytes(['foo'] * 1_000, 100) == ['foo'] * 5 - assert truncate_object_bytes(['foo'] * 1_000_000, 100) == ['foo'] * 5 - assert truncate_object_bytes(['foo'] * 1_000_000, 200) == ['foo'] * 14 - assert truncate_object_bytes([], 200) == [] + assert common.truncate_object_bytes(['foo'] * 10, 100) == ['foo'] * 5 + assert common.truncate_object_bytes(['foo'] * 1_000, 100) == ['foo'] * 5 + assert common.truncate_object_bytes(['foo'] * 1_000_000, 100) == ['foo'] * 5 + assert common.truncate_object_bytes(['foo'] * 1_000_000, 200) == ['foo'] * 14 + assert common.truncate_object_bytes([], 200) == [] - assert truncate_object_bytes(None, 100) is None - assert truncate_object_bytes('', 100) == '' + assert common.truncate_object_bytes(None, 100) is None + assert common.truncate_object_bytes('', 100) == '' - assert truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof' - assert truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo' - assert truncate_object_bytes('foo' * 100, 55) == 'foofo' - assert truncate_object_bytes('foo' * 100, 51) == 'f' - assert truncate_object_bytes('foo' * 100, 50) == '' - assert truncate_object_bytes('foo' * 100, 0) == '' + assert common.truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof' + assert common.truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo' + assert common.truncate_object_bytes('foo' * 100, 55) == 'foofo' + assert common.truncate_object_bytes('foo' * 100, 51) == 'f' + assert common.truncate_object_bytes('foo' * 100, 50) == '' + assert common.truncate_object_bytes('foo' * 100, 0) == '' def test_check_media_directory(test_directory): """The directory provided by the test_directory fixture is a valid media directory.""" - assert check_media_directory() is True + assert common.check_media_directory() is True def test_bad_check_media_directory(): """/dev/full is not a valid media directory, warnings are issued and the check fails.""" with mock.patch('wrolpi.common.get_media_directory') as mock_get_media_directory: mock_get_media_directory.return_value = pathlib.Path('/dev/full') - assert check_media_directory() is False + assert common.check_media_directory() is False + + +def test_chunks_by_name(test_directory, make_files_structure): + """`chunks_by_name` breaks a list of paths on the name change close to the size.""" + with pytest.raises(ValueError): + assert list(common.chunks_by_name([], 0)) == [[]] + + assert list(common.chunks_by_name([], 5)) == [[]] + assert list(common.chunks_by_name([1, 2, 3], 5)) == [[1, 2, 3]] + + files = make_files_structure([ + 'foo.mp4', 'foo.txt', 'foo.png', 'foo.info.json', + 'bar.mp4', 'bar.readability.txt', 'bar.jpeg', 'bar.info.json', + 'baz.mp4', 'baz.txt', 'baz.jpg', 'baz.info.json', + 'qux.mp4', 'qux.txt', 'qux.tif', 'qux.info.json', + ]) + + def assert_chunks(size, files_, expected): + for chunk, expected_chunk in zip_longest(list(common.chunks_by_name(files_, size)), expected): + assert chunk == [test_directory / i for i in expected_chunk] + + assert_chunks(8, files, [ + ['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt', + 'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt', + 'foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'], + ['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'], + ]) + + assert_chunks(6, files, [ + ['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt', + 'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'], + ['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt', + 'qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'], + ]) + + assert_chunks(5, files, [ + ['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt', + 'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'], + ['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt', + 'qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'], + ]) + + assert_chunks(3, files, [ + ['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt'], + ['baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'], + ['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'], + ['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'], + ]) + + assert_chunks(1, files, [ + ['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt'], + ['baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'], + ['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'], + ['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'], + ]) + + files = make_files_structure(['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']) + assert_chunks(1, files, [['1.mp4', '1.txt'], ['2.mp4', '2.png', '2.txt'], ['3.mp4']]) + assert_chunks(3, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt'], ['3.mp4']]) + assert_chunks(6, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt', '3.mp4']]) + # List is not sorted because it is less than the size. + assert_chunks(20, files, [['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']]) + + +@pytest.mark.parametrize( + 'value,expected', [ + ('true', True), + ('t', True), + ('True', True), + ('1', True), + ('yes', True), + ('no', False), + ('0', False), + ('False', False), + ('false', False), + ('f', False), + ('other', False), + ('trust', False), + ('', False), + (None, False), + ] +) +def test_truthy_arg(value, expected): + assert wrolpi.vars.truthy_arg(value) is expected, f'{value} != {expected}' diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 14decddb..a950b69a 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -314,6 +314,7 @@ def test_get_status(test_client, test_session): assert 'hotspot_status' in response.json and isinstance(response.json['hotspot_status'], str) assert 'throttle_status' in response.json and isinstance(response.json['throttle_status'], str) assert 'version' in response.json and isinstance(response.json['version'], str) + assert 'memory_stats' in response.json and isinstance(response.json['memory_stats'], dict) def test_post_download(test_session, test_client, test_download_manager_config): diff --git a/wrolpi/test/test_status.py b/wrolpi/test/test_status.py index 8e790a84..d6cd52cf 100644 --- a/wrolpi/test/test_status.py +++ b/wrolpi/test/test_status.py @@ -59,22 +59,26 @@ async def test_get_drives_info(): @pytest.mark.asyncio async def test_get_bandwidth_info(): """Bandwidth requires psutil""" - bandwidths = await status.get_bandwidth_info() - assert bandwidths == [] + nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info() + assert nic_bandwidths == [] + assert disk_bandwidths == [] await status.bandwidth_worker(2) - bandwidths = await status.get_bandwidth_info() - assert isinstance(bandwidths, list) - assert len(bandwidths) > 0 - assert isinstance(bandwidths[0], status.BandwidthInfo) + nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info() + assert isinstance(nic_bandwidths, list) + assert len(nic_bandwidths) > 0 + assert isinstance(nic_bandwidths[0], status.NICBandwidthInfo) + assert isinstance(disk_bandwidths, list) + assert len(disk_bandwidths) > 0 + assert isinstance(disk_bandwidths[0], status.DiskBandwidthInfo) with mock.patch('wrolpi.status.psutil.net_io_counters') as mock_net_io_counters: # NO FALLBACK! mock_net_io_counters.side_effect = Exception('testing no psutil') await status.bandwidth_worker(1) - bandwidths = await status.get_bandwidth_info() - assert isinstance(bandwidths[0].bytes_recv, int) - assert isinstance(bandwidths[0].bytes_sent, int) - assert isinstance(bandwidths[0].elapsed, int) - assert isinstance(bandwidths[0].speed, int) - assert isinstance(bandwidths[0].name, str) + nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info() + assert isinstance(nic_bandwidths[0].bytes_recv, int) + assert isinstance(nic_bandwidths[0].bytes_sent, int) + assert isinstance(nic_bandwidths[0].elapsed, int) + assert isinstance(nic_bandwidths[0].speed, int) + assert isinstance(nic_bandwidths[0].name, str) diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 8f35ce32..69e834bd 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -1,3 +1,4 @@ +import multiprocessing import os import sys from pathlib import Path @@ -15,6 +16,33 @@ DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else F # tests are running PYTEST = 'pytest' in sys.modules + +def truthy_arg(value: str) -> bool: + """Attempts to convert `value` from console arguments to a boolean value. + + >>> truthy_arg('true') + True + >>> truthy_arg('True') + True + >>> truthy_arg('t') + True + >>> truthy_arg('1') + True + >>> truthy_arg('yes') + True + >>> truthy_arg('0') + False + >>> truthy_arg('false') + False + >>> truthy_arg('f') + False + >>> truthy_arg('no') + False + """ + v = str(value).lower() + return v == 'true' or v == 't' or v == '1' or v == 'yes' + + # Get the media directory from the environment. DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi') MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY)) @@ -37,3 +65,12 @@ DB_PORT = int(os.environ.get('DB_PORT', 5432)) DB_NAME = os.environ.get('DB_NAME', 'wrolpi') DB_USER = os.environ.get('DB_USER', 'wrolpi') DB_PASSWORD = os.environ.get('DB_PASSWORD', 'wrolpi') + +API_HOST = os.environ.get('API_HOST', '127.0.0.1') +API_PORT = os.environ.get('API_PORT', '8081') +API_WORKERS = os.environ.get('API_WORKERS', multiprocessing.cpu_count()) +API_AUTO_RELOAD = truthy_arg(os.environ.get('API_AUTO_RELOAD', DOCKERIZED)) +API_ACCESS_LOG = truthy_arg(os.environ.get('API_ACCESS_LOG', DOCKERIZED)) +API_DEBUG = truthy_arg(os.environ.get('API_DEBUG', False)) + +FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100)) diff --git a/wrolpi/version.py b/wrolpi/version.py index 0d548944..8c4df3eb 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.7.0-beta' +__version__ = '0.7.1-beta' import subprocess