Stopping renderd when reseting map db.

Skipping channel import during test unless explicitly requested.

Deleting gis database during upgrade, if user agrees.  Increasing API verbosity.

Using status context to prevent fetching status in useHotspot.

Disabling wifi toggle button when pending.

Toggling hotspot when clicking nav hotspot icon.

Displaying Hotspot status icon in nav menu.

Importing channels/favorites after refreshing.

Refresh tasks cancel when server shuts down.

Bumping version for release speed.

Refreshing files is 3x faster.

Creating useRecurringTimeout hook which repeatedly calls a callback after a timeout.

Changing intervals to timeout so fetches don't stack up while server is loaded.

Checking that video file exists before fetching.

Creating release candidate branch.  Not indexing model associated files.
This commit is contained in:
Roland 2022-10-19 09:18:53 -06:00
parent dbdcc7a57c
commit 6fa6be50fe
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
32 changed files with 878 additions and 358 deletions

View file

@ -1,5 +1,5 @@
import React, {useContext, useEffect, useState} from "react";
import {Card, Container, Image, Input, Pagination, Responsive, TableCell, TableRow} from 'semantic-ui-react';
import {Card, Container, IconGroup, Image, Input, Pagination, Responsive, TableCell, TableRow} from 'semantic-ui-react';
import {Link, NavLink} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks";
@ -734,6 +734,24 @@ export function DarkModeToggle() {
</>
}
export function HotspotStatusIcon() {
const {on, setHotspot} = useHotspot();
const toggleHotspot = (e) => {
e.preventDefault();
if (on != null) {
setHotspot(!on);
}
}
return <a href='#' onClick={toggleHotspot}>
<IconGroup size='large'>
<Icon name='wifi' disabled={on !== true}/>
{on === false && <Icon corner name='x'/>}
{on === null && <Icon corner name='question'/>}
</IconGroup>
</a>
}
export function useTitle(title) {
const documentDefined = typeof document !== 'undefined';
const originalTitle = React.useRef(documentDefined ? document.title : null);

View file

@ -3,7 +3,7 @@ import {NavLink} from "react-router-dom";
import {Dropdown, Menu, Responsive} from "semantic-ui-react";
import {StatusContext, ThemeContext} from "../contexts/contexts";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {DarkModeToggle} from "./Common";
import {DarkModeToggle, HotspotStatusIcon} from "./Common";
const responsiveWidth = 500;
@ -63,6 +63,10 @@ function MenuLink(props) {
}
}
function NavIcon(props) {
return <div style={{margin: '0.8em'}}>{props.children}</div>
}
export function NavBar() {
const {status} = useContext(StatusContext);
const wrol_mode = status ? status['wrol_mode'] : null;
@ -81,17 +85,15 @@ export function NavBar() {
)
})}
<Responsive minWidth={responsiveWidth} as={Menu.Menu} position="right">
<div style={{margin: '0.8em'}}>
<DarkModeToggle/>
</div>
<NavIcon><HotspotStatusIcon/></NavIcon>
<NavIcon><DarkModeToggle/></NavIcon>
{rightLinks.map((link) => <MenuLink link={link} key={link.key}/>)}
</Responsive>
{/*Show the menu items in a dropdown when on mobile*/}
<Responsive as={Menu.Menu} maxWidth={responsiveWidth - 1} position='right'>
<div style={{margin: '0.8em'}}>
<DarkModeToggle/>
</div>
<NavIcon><HotspotStatusIcon/></NavIcon>
<NavIcon><DarkModeToggle/></NavIcon>
<Dropdown item icon="bars">
<Dropdown.Menu>
{collapsedLinks.map((link) =>

View file

@ -169,7 +169,7 @@ function Statistics() {
}
const videoNames = [
{key: 'videos', label: 'Downloaded Videos'},
{key: 'videos', label: 'Videos'},
{key: 'favorites', label: 'Favorite Videos'},
{key: 'sum_size', label: 'Total Size'},
{key: 'max_size', label: 'Largest Video'},

View file

@ -1,10 +1,10 @@
import {Header, Loader, Progress, Statistic, StatisticGroup} from "../Theme";
import React from "react";
import {Header, Progress, Statistic, StatisticGroup} from "../Theme";
import React, {useContext} from "react";
import {humanBandwidth, humanFileSize, LoadStatistic} from "../Common";
import {getStatus} from "../../api";
import {Container, Divider} from "semantic-ui-react";
import {Divider} from "semantic-ui-react";
import {ProgressPlaceholder} from "../Placeholder";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {StatusContext} from "../../contexts/contexts";
function DriveInfo({used, size, percent, mount}) {
let color;
@ -102,75 +102,64 @@ export function CPUUsageProgress({value, label}) {
return <Progress percent={value} progress color={color} label={label}/>
}
export class Status extends React.Component {
export function Status() {
const {status} = useContext(StatusContext);
constructor(props) {
super(props);
this.state = {
status: null,
}
this.fetchStatus = this.fetchStatus.bind(this);
let percent;
let cores;
let temperature;
let high_temperature;
let critical_temperature;
let minute_1;
let minute_5;
let minute_15;
let bandwidth;
let drives = [];
if (status && status['cpu_info']) {
const {cpu_info, load} = status;
percent = cpu_info['percent'];
cores = cpu_info['cores'];
temperature = cpu_info['temperature'];
high_temperature = cpu_info['high_temperature'];
critical_temperature = cpu_info['critical_temperature'];
minute_1 = load['minute_1'];
minute_5 = load['minute_5'];
minute_15 = load['minute_15'];
drives = status['drives'];
bandwidth = status['bandwidth'];
}
async componentDidMount() {
await this.fetchStatus();
this.intervalId = setInterval(this.fetchStatus, 1000 * 5);
}
return <>
<CPUUsageProgress value={percent} label='CPU Usage'/>
componentWillUnmount() {
clearInterval(this.intervalId);
}
<CPUTemperatureStatistic
value={temperature}
high_temperature={high_temperature}
critical_temperature={critical_temperature}
/>
<Statistic label='Cores' value={cores || '?'}/>
fetchStatus = async () => {
try {
let status = await getStatus();
this.setState({status: status});
} catch (e) {
console.error(e);
this.setState({status: null});
}
}
<Divider/>
render() {
if (this.state.status) {
const {cpu_info, load, drives, bandwidth} = this.state.status;
const {minute_1, minute_5, minute_15} = load;
const {percent, cores, temperature, high_temperature, critical_temperature} = cpu_info;
<StatisticGroup>
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
</StatisticGroup>
return <>
<CPUUsageProgress value={percent} label='CPU Usage'/>
<Divider/>
<CPUTemperatureStatistic
value={temperature}
high_temperature={high_temperature}
critical_temperature={critical_temperature}
/>
<Statistic label='Cores' value={cores || '?'}/>
<Header as='h1'>Bandwidth</Header>
{bandwidth ?
bandwidth.map(i => <BandwidthProgressGroup key={i['name']} bandwidth={i}/>) :
<ProgressPlaceholder/>}
<Divider/>
<Divider/>
<StatisticGroup>
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
</StatisticGroup>
<Divider/>
<Header as='h1'>Bandwidth</Header>
{bandwidth ?
bandwidth.map(i => <BandwidthProgressGroup key={i['name']} bandwidth={i}/>) :
<ProgressPlaceholder/>}
<Divider/>
<Header as='h1'>Drive Usage</Header>
{drives.map((drive) => <DriveInfo key={drive['mount']} {...drive}/>)}
</>
}
return <Container fluid style={{marginBottom: '5em'}}>
<Loader active inline='centered' size='big'>Loading system status</Loader>
</Container>
}
<Header as='h1'>Drive Usage</Header>
{drives.map((drive) => <DriveInfo key={drive['mount']} {...drive}/>)}
</>
}

View file

@ -1,4 +1,4 @@
import {useEffect, useState} from "react";
import {useContext, useEffect, useRef, useState} from "react";
import {
favoriteVideo,
fetchDomains,
@ -27,6 +27,7 @@ import {
import {createSearchParams, useSearchParams} from "react-router-dom";
import {toast} from "react-semantic-toasts";
import {enumerate, humanFileSize, secondsToFullDuration} from "../components/Common";
import {StatusContext} from "../contexts/contexts";
const calculatePage = (offset, limit) => {
return offset && limit ? Math.round((offset / limit) + 1) : 1;
@ -36,6 +37,25 @@ const calculateTotalPages = (total, limit) => {
return total && limit ? Math.round(total / limit) + 1 : 1;
}
export const useRecurringTimeout = (callback, delay) => {
const timer = useRef(null);
useEffect(() => {
const repeat = async () => {
// Recursively call the callback.
await callback();
timer.current = window.setTimeout(repeat, delay);
}
// Call the repeating function instantly to fetch data fast.
repeat();
// Clear the timeout on unload.
return () => clearTimeout(timer.current);
}, [delay])
}
export const useDomains = () => {
const [domains, setDomains] = useState(null);
const [total, setTotal] = useState(null);
@ -441,34 +461,24 @@ export const useBrowseFiles = () => {
export const useHotspot = () => {
const [on, setOn] = useState(null);
const fetchHotspotStatus = async () => {
let status;
try {
status = await getHotspotStatus();
} catch (e) {
console.error(e);
}
if (status === 'connected') {
// Hotspot is on.
setOn(true);
} else if (status === 'disconnected' || status === 'unavailable') {
// Hotspot can be turned on.
setOn(false);
} else {
// Hotspot is not supported. API is probably running in a Docker container.
setOn(null);
}
}
const {status} = useContext(StatusContext);
useEffect(() => {
fetchHotspotStatus();
}, []);
if (status && status['hotspot_status']) {
const {hotspot_status} = status;
if (hotspot_status === 'connected') {
setOn(true);
} else if (hotspot_status === 'disconnected' || hotspot_status === 'unavailable') {
setOn(false);
} else {
setOn(null);
}
}
}, [status]);
const localSetHotspot = async (on) => {
setOn(null);
await setHotspot(on);
await fetchHotspotStatus();
}
return {on, setOn, setHotspot: localSetHotspot};
@ -488,13 +498,7 @@ export const useDownloads = () => {
}
}
useEffect(() => {
fetchDownloads();
const interval = setInterval(() => {
fetchDownloads();
}, 1000 * 3);
return () => clearInterval(interval);
}, []);
useRecurringTimeout(fetchDownloads, 3 * 1000);
return {onceDownloads, recurringDownloads, fetchDownloads}
}
@ -616,12 +620,7 @@ export const useSettings = () => {
export const useSettingsInterval = () => {
const {settings, fetchSettings} = useSettings();
useEffect(() => {
const interval = setInterval(() => {
fetchSettings();
}, 1000 * 3);
return () => clearInterval(interval);
}, []);
useRecurringTimeout(fetchSettings(), 1000 * 3);
return {settings, fetchSettings};
}
@ -644,27 +643,13 @@ export const useStatus = () => {
}
}
useEffect(() => {
fetchStatus();
const interval = setInterval(() => {
// console.log('fetch');
fetchStatus();
}, 1000 * 5);
return () => clearInterval(interval);
}, []);
return {status, fetchStatus}
}
export const useStatusInterval = () => {
const {status, fetchStatus} = useStatus();
useEffect(() => {
const interval = setInterval(() => {
fetchStatus();
}, 1000 * 3);
return () => clearInterval(interval);
}, []);
useRecurringTimeout(fetchStatus, 1000 * 3);
return {status, fetchStatus};
}

View file

@ -5,7 +5,7 @@ PartOf=wrolpi.target
[Service]
Environment=PYTHONUNBUFFERED=1
ExecStart=/usr/bin/python3 /opt/wrolpi/main.py api
ExecStart=/usr/bin/python3 /opt/wrolpi/main.py -v api
Group=wrolpi
User=wrolpi
WorkingDirectory=/opt/wrolpi/

View file

@ -1,7 +1,6 @@
#! /usr/bin/env python3
import argparse
import logging
import os
import sys
import pytz
@ -12,6 +11,7 @@ from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin
from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, wrol_mode_enabled
from wrolpi.dates import set_timezone
from wrolpi.downloader import download_manager
from wrolpi.files.lib import cancel_refresh_tasks
from wrolpi.root_api import api_app
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
from wrolpi.version import get_version_string
@ -214,10 +214,11 @@ async def bandwidth_worker(app: Sanic):
@root_api.api_app.signal(Event.SERVER_SHUTDOWN_BEFORE)
@limit_concurrent(1)
def handle_server_shutdown(*args, **kwargs):
async def handle_server_shutdown(*args, **kwargs):
"""Stop downloads when server is shutting down."""
if not PYTEST:
download_manager.stop()
await cancel_refresh_tasks()
if __name__ == '__main__':

View file

@ -1,4 +1,3 @@
import itertools
import json
import pathlib
from abc import ABC
@ -63,10 +62,10 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session):
if any(lib.is_singlefile_file(i.path) for i in group):
archive_groups[stem] = group
count = itertools.cycle(range(101))
for stem, group in archive_groups.items():
readability_file, singlefile_file, readability_json_file, readability_txt_file, screenshot_file = \
match_archive_files(group)
archive: Archive = Archive.find_by_path(singlefile_file.path, session)
if not archive:
archive = Archive(singlefile_file=singlefile_file)
@ -78,16 +77,16 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session):
if archive.readability_file:
archive.readability_file.associated = True
archive.readability_file.do_index()
archive.readability_file.do_stats()
if archive.readability_json_file:
archive.readability_json_file.associated = True
archive.readability_json_file.do_index()
archive.readability_json_file.do_stats()
if archive.readability_txt_file:
archive.readability_txt_file.associated = True
archive.readability_txt_file.do_index()
archive.readability_txt_file.do_stats()
if archive.screenshot_file:
archive.screenshot_file.associated = True
archive.screenshot_file.do_index()
archive.screenshot_file.do_stats()
archive.singlefile_file.model = Archive.__tablename__
@ -97,10 +96,6 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session):
# Remove this group, it will not be processed by other modelers.
del groups[stem]
if next(count) == 100:
# Commit occasionally.
session.commit()
@register_after_refresh
@limit_concurrent(1)
@ -130,15 +125,14 @@ def archive_cleanup():
with get_db_session(commit=True) as session:
# Remove any Domains without any Archives.
domain_ids = [i[0] for i in session.execute('SELECT DISTINCT domain_id FROM archive') if i[0]]
deleted = 0
for domain in session.query(Domain):
if domain.id not in domain_ids:
deleted += 1
session.delete(domain)
@register_indexer('text/html')
class ArchiveIndexer(Indexer, ABC):
"""Gathers index data from the files associated with an Archive."""
@staticmethod
def get_title(file: File):

View file

@ -12,6 +12,7 @@ from sqlalchemy.orm.collections import InstrumentedList
from wrolpi.common import ModelHelper, Base, logger
from wrolpi.dates import TZDateTime
from wrolpi.errors import InvalidArchive
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.files.models import File
from wrolpi.media_path import MediaPathType
@ -78,6 +79,7 @@ class Archive(Base, ModelHelper):
return self.archive_datetime > other.archive_datetime
def __json__(self):
stem, _ = split_path_stem_and_suffix(self.singlefile_path) if self.singlefile_path else (None, None)
d = dict(
archive_datetime=self.archive_datetime,
domain=self.domain.dict() if self.domain else None,
@ -88,7 +90,7 @@ class Archive(Base, ModelHelper):
readability_txt_path=self.readability_txt_path,
screenshot_path=self.screenshot_path,
singlefile_path=self.singlefile_path,
stem=self.singlefile_path.stem if self.singlefile_path else None,
stem=stem,
title=self.title,
url=self.url,
)
@ -172,7 +174,8 @@ class Archive(Base, ModelHelper):
readability_json_file = self.readability_json_file
if not readability_json_file or not readability_json_file.path.is_file():
raise InvalidArchive(f'Archive must have a readability json file: {self.singlefile_file.path}')
logger.warning(f'Archive must have a readability json file: {self.singlefile_file.path}')
return
try:
with readability_json_file.path.open() as fh:

View file

@ -12,6 +12,7 @@ from modules.archive.lib import get_or_create_domain, get_new_archive_files, del
from modules.archive.models import Archive, Domain
from wrolpi.dates import local_timezone
from wrolpi.db import get_db_session
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import File
from wrolpi.root_api import CustomJSONEncoder
from wrolpi.test.common import skip_circleci
@ -430,6 +431,43 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
assert response.json['files'][0]['archive']['archive_datetime'], 'Archive has no datetime'
@pytest.mark.asyncio
async def test_refresh_archives_index(test_session, test_directory, test_client, make_files_structure):
"""Archives are indexed using ArchiveIndexer and archive_modeler."""
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: https://example.com
saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">'''
singlefile, *_ = make_files_structure({
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
'archive/example.com/2021-10-05-16-20-10_NA.png': None,
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents',
'archive/example.com/2021-10-05-16-20-10_NA.readability.json':
'{"url": "https://example.com", "title": "the title"}',
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
})
await refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
assert archive.singlefile_file.a_text == 'the title'
assert archive.singlefile_file.d_text == 'article text contents'
# The associated files are not indexed.
for name in ('screenshot_file', 'readability_file', 'readability_json_file', 'readability_txt_file'):
file = getattr(archive, name)
assert file.associated is True, 'The associated file is not marked'
assert file.indexed is False
# file.a_text is the file name.
assert not file.b_text
assert not file.c_text
assert not file.d_text
def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure):
"""
Invalid Archives should be removed during refresh.

View file

@ -1,4 +1,3 @@
import itertools
import json
import pathlib
from abc import ABC
@ -22,12 +21,10 @@ __all__ = ['video_modeler', 'VideoIndexer']
@register_modeler
def video_modeler(groups: Dict[str, List[File]], session: Session):
count = itertools.cycle(range(101))
# Only process groups that contain a video file.
local_groups = groups.copy()
videos = []
for stem, group in local_groups.items():
video_file = next((i for i in group if i.mimetype.split('/')[0] == 'video' and i.path.suffix != '.part'), None)
video_file = next((i for i in group if i.mimetype.startswith('video/') and i.path.suffix != '.part'), None)
if not video_file:
# Not a video group.
continue
@ -40,28 +37,34 @@ def video_modeler(groups: Dict[str, List[File]], session: Session):
if poster_file:
poster_file.associated = True
poster_file.do_index()
poster_file.do_stats()
if caption_file:
caption_file.associated = True
caption_file.do_index()
caption_file.do_stats()
if info_json_file:
info_json_file.associated = True
info_json_file.do_index()
info_json_file.do_stats()
video: Video = Video.upsert(video_file, session)
video = session.query(Video).filter_by(video_file=video_file).one_or_none()
if not video:
video = Video(video_file=video_file)
session.add(video)
video.poster_file = poster_file
video.caption_file = caption_file
video.info_json_file = info_json_file
video.size = video_file.path.stat().st_size
video_file.model = Video.__tablename__
video.video_file.do_index()
video.validate(session)
videos.append(video)
# Remove this group, it will not be processed by other modelers.
del groups[stem]
if videos:
session.flush(videos)
for video in videos:
video.video_file.do_index()
video.validate(session)
if next(count) == 100:
# Commit occasionally.
session.commit()
# Remove this group, it will not be processed by other modelers.
del groups[video.video_path.stem]
@register_after_refresh

View file

@ -7,6 +7,7 @@ from modules.videos.channel import lib
from modules.videos.lib import save_channels_config, import_channels_config
from modules.videos.models import Channel, Video
from wrolpi.errors import UnknownChannel
from wrolpi.files.lib import refresh_files
@pytest.mark.parametrize('params', [
@ -93,7 +94,8 @@ def test_channels_no_url(test_session, test_directory):
import_channels_config()
def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory):
@pytest.mark.asyncio
async def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory):
"""A Video will be assigned to a Channel by its info json first, then to its directory."""
channel1 = channel_factory(source_id='channel1')
channel2 = channel_factory(source_id='channel2')
@ -110,20 +112,20 @@ def test_refresh_videos_finds_channel(test_session, test_client, channel_factory
with mock.patch('wrolpi.common.after_refresh') as mock_after_refresh:
# Channel can be found in `video_cleanup`, prevent that with mock.
mock_after_refresh.side_effect = []
test_client.post('/api/files/refresh')
await refresh_files()
# Channel in info_json was found during validation.
assert vid1.channel_id == channel1.id
# Refresh using cleanup, info json should be trusted.
test_client.post('/api/files/refresh')
await refresh_files()
assert vid1.channel_id == channel1.id
# Remove the info json, the Video's Channel will be its directory.
vid1.channel_id = None
vid1.info_json_path.unlink()
test_session.commit()
test_client.post('/api/files/refresh')
await refresh_files()
test_session.refresh(vid1)
vid1 = test_session.query(Video).one()
assert vid1.channel_id == channel2.id

View file

@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi import before_startup
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, after_refresh, register_after_refresh
from wrolpi.dates import from_timestamp, Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.files.models import File
@ -375,8 +375,14 @@ def save_channels_config(session=None, preserve_favorites: bool = True):
@before_startup
@register_after_refresh
def import_channels_config():
"""Import channel settings to the DB. Existing channels will be updated."""
if PYTEST and not TEST_CHANNELS_CONFIG:
logger.warning(f'Not importing channels during this test. Use `test_channels_config` fixture if you would'
f'like to call this.')
return
logger.info('Importing videos config')
try:
config = get_channels_config()

View file

@ -13,7 +13,7 @@ from wrolpi.dates import now, TZDateTime
from wrolpi.db import get_db_curs
from wrolpi.downloader import Download, download_manager
from wrolpi.errors import UnknownVideo
from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem
from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import File
from wrolpi.media_path import MediaPathType
@ -240,7 +240,7 @@ class Video(ModelHelper, Base):
def __json__(self):
from modules.videos.common import minimize_video_info_json
info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None
stem = self.video_path.stem if self.video_path else None
stem, _ = split_path_stem_and_suffix(self.video_path) if self.video_file else (None, None)
d = dict(
caption_path=self.caption_path,

View file

@ -3,13 +3,41 @@ import shutil
from http import HTTPStatus
from pathlib import Path
import pytest
from modules.videos.models import Video, Channel
from modules.videos.video import lib as video_lib
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import File
from wrolpi.vars import PROJECT_DIR
@pytest.mark.asyncio
async def test_refresh_videos_index(test_session, test_directory, video_factory):
"""A video discovered during refresh has files associated with it. Only the video file is indexed."""
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
test_session.commit()
test_session.query(Video).delete()
test_session.query(File).delete()
test_session.commit()
for _ in range(1):
# Refreshing twice does not change the results.
await refresh_files()
assert test_session.query(File).count() == 4
video: Video = test_session.query(Video).one()
assert video.video_file.indexed is True, 'Video was not indexed'
assert video.video_file.a_text, 'Video title was not indexed'
assert video.video_file.d_text, 'Video captions were not indexed'
assert not video.poster_file.indexed
assert not video.caption_file.indexed
assert not video.info_json_file.indexed
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
subdir = test_directory / 'subdir'
subdir.mkdir()

View file

@ -70,14 +70,14 @@ def test_parse_video_file_name(file_name, expected):
@pytest.mark.asyncio
async def test_associated_files(test_session, test_directory, video_file, image_file, vtt_file1):
async def test_associated_video_files(test_session, test_directory, video_file, image_file, vtt_file1):
"""A Video file has files related to it. They should be attached to the Video record."""
caption_file = video_file.with_suffix('.en.vtt')
vtt_file1.rename(caption_file)
poster_file = video_file.with_suffix('.jpeg')
image_file.rename(poster_file)
await files_lib._refresh_all_files()
await files_lib.refresh_files()
test_session.commit()
# All files were found.
@ -93,7 +93,7 @@ async def test_associated_files(test_session, test_directory, video_file, image_
assert video.video_file.size
assert video.video_file.indexed
await files_lib._refresh_all_files()
await files_lib.refresh_files()
test_session.commit()
# No new files were added.
@ -113,7 +113,7 @@ async def test_unassociated_video(test_session, test_directory, video_file):
video_file.rename(new_video_file_path)
video_file = new_video_file_path
await files_lib._refresh_all_files()
await files_lib.refresh_files()
test_session.commit()
video: Video = test_session.query(Video).one()

View file

@ -34,8 +34,8 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
caption = video.video_file.d_text
video = video.video_file.__json__()
video['video']['caption'] = caption
previous_video = previous_video.video_file.__json__() if previous_video else None
next_video = next_video.video_file.__json__() if next_video else None
previous_video = previous_video.video_file.__json__() if previous_video and previous_video.video_file else None
next_video = next_video.video_file.__json__() if next_video and next_video.video_file else None
return video, previous_video, next_video

View file

@ -22,12 +22,12 @@ fi
if [[ ${1} == *.osm.pbf ]]; then
# Import a PBF file.
osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
nice -n 18 osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 3 \
-S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}"
elif [[ ${1} == *.dump ]]; then
# Import a Postgresql dump.
pg_restore -j4 --no-owner -d gis "${1}"
nice -n 18 pg_restore -j3 --no-owner -d gis "${1}"
else
echo "Cannot import unknown map file"
exit 5

View file

@ -18,6 +18,21 @@ apt install -y libboost-all-dev git tar unzip wget bzip2 build-essential autocon
/usr/bin/npm install -g carto
/usr/bin/carto -v
function yes_or_no {
while true; do
read -p "$* [y/n]:" yn
case $yn in
[Yy]*) return 0 ;;
[Nn]*) return 1 ;;
esac
done
}
sudo -u postgres psql -c '\l' | grep gis &&
yes_or_no "Map database already exists. Replace it?" &&
systemctl stop renderd &&
sudo -u postgres dropdb gis
# Create `gis` database for map.
sudo -u postgres psql -c '\l' | grep gis || sudo -u postgres createdb -E UTF8 -O wrolpi gis
sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" || :
@ -64,7 +79,7 @@ while [ "${COUNT}" -lt "${MAX_TRIES}" ]; do
external_data=true
[[ "${dc_import}" = false ]] &&
# Run import in "create" mode so basic tables will be created.
sudo -u wrolpi osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \
sudo -u wrolpi nice -n 18 osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
-S /opt/openstreetmap-carto/openstreetmap-carto.style /tmp/district-of-columbia-latest.osm.pbf &&
dc_import=true

View file

@ -1,4 +1,5 @@
import asyncio
import atexit
import contextlib
import inspect
import logging
@ -26,7 +27,7 @@ from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from wrolpi.dates import DEFAULT_TIMEZONE
from wrolpi.dates import DEFAULT_TIMEZONE, now
from wrolpi.errors import WROLModeEnabled, NativeOnly
from wrolpi.vars import PYTEST, MODULES_DIR, \
DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
@ -37,6 +38,58 @@ formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(messag
ch.setFormatter(formatter)
logger.addHandler(ch)
__all__ = [
'Base',
'ModelHelper',
'get_model_by_table_name',
'tsvector',
'compile_tsvector',
'sanitize_link',
'ConfigFile',
'get_config',
'set_test_config',
'wrol_mode_enabled',
'wrol_mode_check',
'enable_wrol_mode',
'disable_wrol_mode',
'insert_parameter',
'iterify',
'date_range',
'remove_whitespace',
'run_after',
'set_test_media_directory',
'get_media_directory',
'check_media_directory',
'get_absolute_media_path',
'get_relative_to_media_directory',
'get_files_and_directories',
'minimize_dict',
'make_media_directory',
'extract_domain',
'import_modules',
'api_param_limiter',
'partition',
'chdir',
'zig_zag',
'walk',
'escape_file_name',
'native_only',
'recursive_map',
'any_extensions',
'aiohttp_post',
'register_modeler',
'apply_modelers',
'register_after_refresh',
'apply_after_refresh',
'match_paths_to_suffixes',
'chunks',
'chunks_by_name',
'timer',
'cum_timer',
'limit_concurrent',
'truncate_object_bytes'
]
# Base is used for all SQLAlchemy models.
Base = declarative_base()
@ -369,11 +422,6 @@ def date_range(start: dt_or_d, end: dt_or_d, steps: int) -> List[dt_or_d]:
return [start + (delta * i) for i in range(steps)]
def chunks(it, size):
it = iter(it)
return iter(lambda: tuple(islice(it, size)), ())
WHITESPACE = re.compile(r'\s')
@ -763,7 +811,7 @@ def apply_modelers(files, session: Session):
for file in files:
if not file.mimetype:
file.mimetype = get_mimetype(file.path)
stem, suffix = split_path_stem_and_suffix(file.path)
stem, _ = split_path_stem_and_suffix(file.path)
try:
groups[stem].append(file)
except KeyError:
@ -771,7 +819,6 @@ def apply_modelers(files, session: Session):
for modeler in modelers:
modeler(groups, session)
session.commit()
for stem, group in groups.items():
# Index any files that were not claimed by modelers.
@ -784,6 +831,7 @@ after_refresh = []
def register_after_refresh(func: callable):
after_refresh.append(func)
return func
def apply_after_refresh():
@ -807,6 +855,56 @@ def match_paths_to_suffixes(paths: List[pathlib.Path], suffix_groups: List[Tuple
yield match
def chunks(it: Iterable, size: int):
"""Split an iterable into iterables of the defined length."""
it = iter(it)
return iter(lambda: tuple(islice(it, size)), ())
def chunks_by_name(it: List[Union[pathlib.Path, str]], size: int) -> Generator[List[pathlib.Path], None, None]:
"""
Attempt to split a list of paths near the defined size. Keep groups of files together when they share
matching names.
>>> files = ['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']
>>> chunks_by_name(files, 2)
[['1.mp4', '1.txt'], ['2.mp4', '2.txt', '2.png'], ['3.mp4']]
>>> chunks_by_name(files, 3)
[['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png'], ['3.mp4']]
"""
if not isinstance(size, int) or size < 1:
raise ValueError('size must be a positive integer')
if not it or len(it) < size:
yield it
return
if not isinstance(it, list):
raise ValueError('iterable must a list')
from wrolpi.files.lib import split_path_stem_and_suffix
it = sorted(it.copy())
index = size
last_name = None
while it:
if index >= len(it):
# Ran out of items, yield what is left.
yield it
return
path = it[index]
name, _ = split_path_stem_and_suffix(path)
if last_name and name != last_name:
# Found a break in the path names, yield and reset.
chunk, it = it[:index], it[index:]
index = size
last_name = None
yield chunk
continue
# Didn't find a name change, try again.
last_name = name
index += 1
@contextlib.contextmanager
def timer(name):
"""Prints out the time elapsed during the call of some block."""
@ -817,6 +915,35 @@ def timer(name):
logger.warning(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
TIMERS = dict()
@contextlib.contextmanager
def cum_timer(name: str):
"""Track time usage within this context's name. The cumulative calls to this context will be printed on exit."""
before = now()
try:
yield
finally:
total_seconds = (now() - before).total_seconds()
existing_seconds, calls = TIMERS.get(name, (0, 0))
TIMERS[name] = (existing_seconds + total_seconds, calls + 1)
@atexit.register
def print_timer():
"""Print any cumulative timers that have been stored during exit."""
if not TIMERS:
return
total = sum(i[0] for i in TIMERS.values())
for name, (seconds, calls) in sorted(TIMERS.items(), key=lambda i: i[1]):
percent = int((seconds / total) * 100)
seconds = round(seconds, 5)
print(f'CUM_TIMER: {repr(name)} elapsed {seconds} cumulative seconds ({percent}%)'
f' {calls} calls', file=sys.stderr)
def limit_concurrent(limit: int):
"""
Wrapper that limits the amount of concurrently running functions.
@ -852,10 +979,6 @@ def limit_concurrent(limit: int):
return wrapper
async def count_files(path: pathlib.Path) -> int:
return len([i for i in walk(path) if i.is_file()])
def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> List[str]:
"""
Shorten an object. This is useful when inserting something into a tsvector.

View file

@ -1,3 +1,4 @@
import logging
from contextlib import contextmanager
from functools import wraps
from typing import ContextManager, Tuple, List, Union, Type
@ -41,7 +42,7 @@ postgres_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{d
# This engine is used for all normal tasks (except testing).
db_args = get_db_args()
connect_args = dict(application_name='wrolpi_api')
connect_args = dict(application_name='wrolpi_api', connect_timeout=600)
uri = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**db_args)
engine = create_engine(uri, poolclass=NullPool, connect_args=connect_args)
session_maker = sessionmaker(bind=engine)
@ -75,6 +76,9 @@ def get_db_context() -> Tuple[create_engine, Session]:
return local_engine, session
# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
@contextmanager
def get_db_session(commit: bool = False) -> ContextManager[Session]:
"""

View file

@ -103,6 +103,8 @@ class TextIndexer(Indexer, ABC):
Detects VTT (caption) files and forwards them on."""
_ignored_suffixes = ('.json', '.vtt', '.srt', '.csv')
@classmethod
def create_index(cls, file):
path = file.path.path if hasattr(file.path, 'path') else file.path
@ -120,10 +122,7 @@ class TextIndexer(Indexer, ABC):
@classmethod
def detect_special_indexer(cls, file):
if file.path.name.endswith('.json'):
# JSON files are detected as "text/plain" but we don't want them to be indexed.
return DefaultIndexer
if file.path.name.endswith('.vtt') or file.path.name.endswith('.srt'):
# Don't process caption files.
if any(file.path.name.endswith(i) for i in cls._ignored_suffixes):
# Some text files aren't for humans.
return DefaultIndexer
return cls

View file

@ -4,6 +4,7 @@ import glob
import pathlib
import re
import subprocess
from asyncio import Task
from functools import wraps
from pathlib import Path
from typing import List, Tuple, Union
@ -13,12 +14,12 @@ from sqlalchemy.orm import Session
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name
get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name, chunks_by_name
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidFile
from wrolpi.files.models import File
from wrolpi.vars import PYTEST
from wrolpi.vars import PYTEST, FILE_REFRESH_CHUNK_SIZE
try:
import magic
@ -88,20 +89,6 @@ def delete_file(file: str):
FILE_NAME_REGEX = re.compile(r'[_ .]')
def split_file_name_into_words(path: Path) -> List[str]:
"""
Split a file name into words.
>>> split_path_stem_and_suffix(Path('foo.mp4'))
['foo']
>>> split_path_stem_and_suffix(Path('foo bar.mp4'))
['foo', 'bar']
"""
words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)]
return words
FILE_BIN = which('file', '/usr/bin/file')
@ -144,55 +131,88 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str]) -> Tuple[str, str
return path.name, ''
refresh_logger = logger.getChild('refresh')
@wrol_mode_check
def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime):
async def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime):
"""Upsert, index, and model all provided paths."""
# Update the idempotency if any files already in the DB.
if not paths:
return
paths = set(paths)
for idx, chunk in enumerate(map(set, chunks_by_name(paths, FILE_REFRESH_CHUNK_SIZE))):
with get_db_session(commit=True) as session:
existing_files = list(session.query(File).filter(File.path.in_(chunk)))
existing_paths = {i.path for i in existing_files}
# Add any new files into the database.
new_files = []
if new_paths := (chunk - existing_paths):
new_files = [File(path=i, idempotency=idempotency, mimetype=get_mimetype(i)) for i in new_paths]
session.add_all(new_files)
session.flush(new_files)
# Apply models to all files. Many files will have no model.
all_files = existing_files + new_files
apply_modelers(all_files, session)
with get_db_session(commit=True) as session:
existing_files = list(session.query(File).filter(File.path.in_(paths)))
for existing_file in existing_files:
existing_file.idempotency = idempotency
session.commit()
# Update idempotency for existing files after modelers. Otherwise SQLAlchemy forgets.
for existing_file in existing_files:
existing_file.idempotency = idempotency
existing_paths = {i.path for i in existing_files}
# Add any new files into the database.
new_files = []
if new_paths := (paths - existing_paths):
new_files = [File(path=i, idempotency=idempotency) for i in new_paths]
session.add_all(new_files)
# Commit before modeling. Just in case modeling fails.
session.commit()
if idx > 0:
parent = all_files[0].path.parent
refresh_logger.debug(f'Committed chunk of {len(chunk)} files in {parent}')
# Apply models to all files. Many files will have no model.
apply_modelers(existing_files + new_files, session)
# Sleep between chunks to catch cancel.
await asyncio.sleep(0)
refresh_logger = logger.getChild('refresh')
async def _refresh_directory_files_recursively(directory, idempotency):
"""
Refresh all files in a directory, recursively refresh all files in subdirectories.
"""
async def _refresh_directory_files_recursively(directory: pathlib.Path, idempotency: datetime):
"""Refresh all files in a directory, recursively refresh all files in subdirectories."""
directories = [directory, ]
while directories:
directory = directories.pop(0)
try:
files, new_directories = get_files_and_directories(directory)
refresh_logger.debug(f'Refreshing {len(files)} files in {directory}')
directories.extend(new_directories)
_refresh_files_list(files, idempotency)
if files:
refresh_logger.info(f'Refreshing {len(files)} files in {directory}')
await _refresh_files_list(files, idempotency)
except Exception as e:
logger.error(f'Failed to refresh files in {directory}', exc_info=e)
refresh_logger.error(f'Failed to refresh files in {directory}', exc_info=e)
except asyncio.CancelledError:
refresh_logger.error(f'Refresh canceled during {directory}')
raise
REFRESH_TASKS: List[Task] = []
async def cancel_refresh_tasks():
"""Cancel all refresh tasks, if any."""
if REFRESH_TASKS:
refresh_logger.warning(f'Canceling {len(REFRESH_TASKS)} refreshes')
for task in REFRESH_TASKS:
task.cancel()
await asyncio.gather(*REFRESH_TASKS)
def cancelable_wrapper(func: callable):
@wraps(func)
async def wrapped(*args, **kwargs):
if PYTEST:
return await func(*args, **kwargs)
task = asyncio.create_task(func(*args, **kwargs))
REFRESH_TASKS.append(task)
return wrapped
@limit_concurrent(1) # Only one refresh at a time.
async def _refresh_all_files():
@wrol_mode_check
@cancelable_wrapper
async def refresh_files():
"""Find, model, and index all files in the media directory."""
refresh_logger.warning('Refreshing all files')
@ -217,18 +237,9 @@ async def _refresh_all_files():
refresh_logger.warning('Done refreshing Files')
@wraps(_refresh_all_files)
@wrol_mode_check
async def refresh_files():
"""Schedule a refresh task if not testing. If testing, do a synchronous refresh."""
if PYTEST:
return await _refresh_all_files()
asyncio.create_task(_refresh_all_files())
@limit_concurrent(1)
@wrol_mode_check
@cancelable_wrapper
async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str]):
"""Upsert and index all files within a directory (recursively).

View file

@ -91,18 +91,7 @@ class File(ModelHelper, Base):
def do_index(self, force_index: bool = False):
"""Gather any missing information about this file. Index the contents of this file using an Indexer."""
from wrolpi.files.lib import split_path_stem_and_suffix
if not self.mimetype:
from wrolpi.files.lib import get_mimetype
self.mimetype = get_mimetype(self.path)
self.title = self.title or self.path.name
_, self.suffix = split_path_stem_and_suffix(self.path)
stat = self.path.stat()
self.size = stat.st_size
self.modification_datetime = from_timestamp(stat.st_mtime)
self.do_stats()
try:
if force_index or self.indexed is not True:
@ -119,6 +108,22 @@ class File(ModelHelper, Base):
except Exception as e:
logger.error(f'Failed to index {self.path}', exc_info=e)
def do_stats(self):
"""Assign the mimetype, title, size, modification_time of this file."""
from wrolpi.files.lib import split_path_stem_and_suffix
if not self.mimetype:
from wrolpi.files.lib import get_mimetype
self.mimetype = get_mimetype(self.path)
self.title = self.title or self.path.name
stat = self.path.stat()
self.size = stat.st_size
self.modification_datetime = from_timestamp(stat.st_mtime)
name, self.suffix = split_path_stem_and_suffix(self.path)
# Use title as the a_text temporarily, this may be overwritten by indexer.
# TODO this is very slow. What can be done instead?
self.a_text = self.a_text or name
@validates('path')
def validate_path(self, key, value):
value = pathlib.Path(value) if not isinstance(value, pathlib.Path) else value

View file

@ -1,8 +1,11 @@
import asyncio
import json
import shutil
import zipfile
from datetime import datetime
from pathlib import Path
from typing import List
from uuid import uuid4
import mock
import pytest
@ -140,21 +143,6 @@ def test_delete_file(make_files_structure, test_directory):
assert (test_directory / 'baz').is_dir()
@pytest.mark.parametrize(
'path,expected',
[
('foo.mp4', ['foo']),
('foo bar.txt', ['foo', 'bar']),
('foo_bar.mp4', ['foo', 'bar']),
('foo-bar', ['foo-bar']),
('123 foo bar', ['123', 'foo', 'bar']),
('123foo bar', ['123foo', 'bar']),
]
)
def test_split_file_name_into_words(path, expected):
assert lib.split_file_name_into_words(Path(path)) == expected
@pytest.mark.parametrize(
'path,expected',
[
@ -197,6 +185,42 @@ async def test_refresh_files(test_session, make_files_structure, test_directory)
assert_files(test_session, [])
@pytest.mark.asyncio
async def test_refresh_many_files(test_session, make_files_structure):
count = 1_000
make_files_structure([f'{uuid4()}.txt' for _ in range(count)])
await lib.refresh_files()
assert test_session.query(File).count() == count
await lib.refresh_files()
assert test_session.query(File).count() == count
@pytest.mark.asyncio
async def test_refresh_cancel(test_session, make_files_structure, test_directory):
"""Refresh tasks can be canceled."""
# Creat a lot of files so the refresh will take too long.
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
async def assert_cancel(task_):
# Time the time it takes to cancel.
before = datetime.now()
# Sleep so the refresh task has time to run.
await asyncio.sleep(0.1)
# Cancel the refresh (it will be sleeping soon).
task_.cancel()
with pytest.raises(asyncio.CancelledError):
await task_
assert (datetime.now() - before).total_seconds() < 0.5, 'Task took too long. Was the refresh canceled?'
task = asyncio.create_task(lib.refresh_files())
await assert_cancel(task)
task = asyncio.create_task(lib.refresh_directory_files_recursively(test_directory))
await assert_cancel(task)
@pytest.mark.asyncio
async def test_refresh_files_in_directory(test_session, make_files_structure, test_directory):
"""A subdirectory can be refreshed, files above it can be ignored."""
@ -276,7 +300,7 @@ async def test_files_indexer(test_session, make_files_structure):
# Enable slow feature for testing.
# TODO can this be sped up to always be included?
with mock.patch('modules.videos.EXTRACT_SUBTITLES', True):
await lib._refresh_all_files()
await lib.refresh_files()
bzip_file, gzip_file, text_file, zip_file, image_file, unknown_file, info_json_file, video_file \
= test_session.query(File).order_by(File.path)
@ -330,7 +354,7 @@ async def test_large_text_indexer(test_session, make_files_structure):
large, = make_files_structure({
'large_file.txt': 'foo ' * 1_000_000,
})
await lib._refresh_all_files()
await lib.refresh_files()
assert test_session.query(File).count() == 1
assert large.is_file() and large.stat().st_size == 4_000_000

View file

@ -1,5 +1,5 @@
import json
import multiprocessing
import json
import re
from datetime import datetime, date
from decimal import Decimal
@ -24,14 +24,14 @@ from wrolpi.dates import set_timezone
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, InvalidTimezone, API_ERRORS, APIError, ValidationError, HotspotError
from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
truthy_arg
from wrolpi.version import __version__
logger = logger.getChild(__name__)
api_app = Sanic(name='api_app')
DEFAULT_HOST, DEFAULT_PORT = '127.0.0.1', '8081'
api_bp = Blueprint('RootAPI', url_prefix='/api')
BLUEPRINTS = [api_bp, ]
@ -50,26 +50,47 @@ def add_blueprint(bp: Union[Blueprint, BlueprintGroup]):
BLUEPRINTS.append(bp)
def run_webserver(host: str, port: int, workers: int = 8):
def run_webserver(
host: str = API_HOST,
port: int = API_PORT,
workers: int = API_WORKERS,
api_debug: bool = API_DEBUG,
access_log: bool = API_ACCESS_LOG,
):
# Attach all blueprints after they have been defined.
for bp in BLUEPRINTS:
api_app.blueprint(bp)
kwargs = dict(host=host, port=port, workers=workers, debug=False, access_log=False, auto_reload=True)
logger.debug(f'Running Sanic {sanic_version} with kwargs {kwargs}')
kwargs = dict(
host=host,
port=port,
workers=workers,
debug=api_debug,
access_log=access_log,
auto_reload=DOCKERIZED,
)
logger.warning(f'Running Sanic {sanic_version} with kwargs {kwargs}')
return api_app.run(**kwargs)
def init_parser(parser):
# Called by WROLPI's main() function
parser.add_argument('-H', '--host', default=DEFAULT_HOST, help='What network interface to connect webserver')
parser.add_argument('-p', '--port', default=DEFAULT_PORT, type=int, help='What port to connect webserver')
parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(), type=int,
help='How many web workers to run')
parser.add_argument('-H', '--host', default=API_HOST, help='What network interface to connect webserver')
parser.add_argument('-p', '--port', default=API_PORT, type=int, help='What port to connect webserver')
parser.add_argument('-w', '--workers', default=API_WORKERS, type=int, help='How many web workers to run')
parser.add_argument('--access-log', default=API_ACCESS_LOG, type=truthy_arg, help='Enable Sanic access log')
parser.add_argument('--api-debug', default=API_DEBUG, type=truthy_arg, help='Enable Sanic debug log')
parser.add_argument('--api-auto-reload', default=API_AUTO_RELOAD, type=truthy_arg, help='Enable Sanic auto reload')
def main(args):
return run_webserver(args.host, args.port, args.workers)
return run_webserver(
host=args.host,
port=args.port,
workers=args.workers,
api_debug=args.api_debug,
access_log=args.access_log,
)
index_html = '''
@ -313,14 +334,21 @@ async def throttle_off(_: Request):
@openapi.description('Get the status of CPU/load/etc.')
async def get_status(_: Request):
s = await status.get_status()
downloads = download_manager.get_summary()
try:
downloads = download_manager.get_summary()
except Exception as e:
logger.error('Unable to get download status', exc_info=e)
downloads = dict()
ret = dict(
bandwidth=s.bandwidth,
cpu_info=s.cpu_info,
disk_bandwidth=s.disk_bandwidth,
downloads=downloads,
drives=s.drives,
hotspot_status=admin.hotspot_status().name,
load=s.load,
memory_stats=s.memory_stats,
throttle_status=admin.throttle_status().name,
version=__version__,
wrol_mode=wrol_mode_enabled(),

View file

@ -1,6 +1,7 @@
#!/usr/bin/env python3
import asyncio
import multiprocessing
import pathlib
import re
import subprocess
from dataclasses import dataclass
@ -9,7 +10,7 @@ from pathlib import Path
from typing import List, Optional, Tuple
from wrolpi.cmd import which
from wrolpi.common import logger
from wrolpi.common import logger, limit_concurrent
from wrolpi.dates import now
try:
@ -41,9 +42,9 @@ class SystemLoad:
def __json__(self):
return dict(
minute_1=str(self.minute_1),
minute_5=str(self.minute_5),
minute_15=str(self.minute_15),
minute_1=str(round(self.minute_1, 2)),
minute_5=str(round(self.minute_5, 2)),
minute_15=str(round(self.minute_15, 2)),
)
@ -179,6 +180,60 @@ async def get_cpu_info() -> CPUInfo:
return info
MEMORY_TOTAL = re.compile(r'^MemTotal:\s+(\d+)\s+kB$', re.MULTILINE)
MEMORY_FREE = re.compile(r'^MemFree:\s+(\d+)\s+kB$', re.MULTILINE)
MEMORY_CACHED = re.compile(r'^Cached:\s+(\d+)\s+kB$', re.MULTILINE)
@dataclass
class MemoryStats:
total: int = None
used: int = None
free: int = None
cached: int = None
def __json__(self):
return dict(
total=self.total,
used=self.used,
free=self.free,
cached=self.cached,
)
def get_memory_stats_psutil():
mem = psutil.virtual_memory()
mem = MemoryStats(
total=mem.total,
used=mem.used,
free=mem.free,
cached=mem.cached,
)
return mem
async def get_memory_stats():
try:
return get_memory_stats_psutil()
except Exception as e:
warn_once(e)
meminfo = pathlib.Path('/proc/meminfo')
if meminfo.is_file() and (contents := meminfo.read_text()):
total = int(i[0]) if (i := MEMORY_TOTAL.findall(contents)) else None
free = int(i[0]) if (i := MEMORY_FREE.findall(contents)) else None
cached = int(i[0]) if (i := MEMORY_CACHED.findall(contents)) else None
mem = MemoryStats(
total=int(total * 1024) if total else None,
free=int(free * 1024) if free else None,
cached=int(cached * 1024) if cached else None,
used=int(total - free) if total and free else None,
)
return mem
return MemoryStats()
@dataclass
class DriveInfo:
mount: str = None
@ -263,7 +318,7 @@ async def get_drives_info() -> List[DriveInfo]:
@dataclass
class BandwidthInfo:
class NICBandwidthInfo:
bytes_recv: int = None
bytes_sent: int = None
elapsed: int = None
@ -288,6 +343,8 @@ IGNORED_NIC_NAMES = {
'br-',
}
BANDWIDTH = multiprocessing.Manager().dict()
def get_nic_names() -> List[str]:
"""Finds all non-virtual and non-docker network interface names."""
@ -299,31 +356,57 @@ def get_nic_names() -> List[str]:
return names
BANDWIDTH = multiprocessing.Manager().dict()
@dataclass
class DiskBandwidthInfo:
bytes_read: int = None
bytes_write: int = None
elapsed: int = None
name: str = None
def __json__(self):
return dict(
bytes_read=self.bytes_read,
bytes_write=self.bytes_write,
elapsed=self.elapsed,
name=self.name,
)
async def get_bandwidth_info() -> List[BandwidthInfo]:
"""
Get all bandwidth information for all NICs.
"""
infos = []
DRIVE_BANDWIDTH = multiprocessing.Manager().dict()
async def get_bandwidth_info() -> Tuple[List[NICBandwidthInfo], List[DiskBandwidthInfo]]:
"""Get all bandwidth information for all NICs."""
nics_info = []
disks_info = []
try:
for name in sorted(BANDWIDTH.keys()):
nic = BANDWIDTH[name]
if 'bytes_recv_ps' not in nic:
# Not stats collected yet.
continue
infos.append(BandwidthInfo(
nics_info.append(NICBandwidthInfo(
bytes_recv=nic['bytes_recv_ps'],
bytes_sent=nic['bytes_sent_ps'],
elapsed=nic['elapsed'],
name=name,
speed=nic['speed'],
))
for name in sorted(DRIVE_BANDWIDTH.keys()):
disk = DRIVE_BANDWIDTH[name]
if 'bytes_read_ps' not in disk:
# No status collected yet
continue
disks_info.append(DiskBandwidthInfo(
bytes_read=disk['bytes_read_ps'],
bytes_write=disk['bytes_write_ps'],
elapsed=disk['elapsed'],
name=name,
))
except Exception as e:
warn_once(e)
return infos
return nics_info, disks_info
def _get_nic_tick(name_):
@ -341,7 +424,7 @@ def _get_nic_tick(name_):
def _calculate_bytes_per_second(history: List[Tuple]) -> Tuple[int, int, int]:
"""Calculate the bytes-per-second between the oldest and newest tick."""
(oldest_now, oldest_recv, oldest_sent, _), (newest_now, newest_recv, newest_sent, _) = \
(oldest_now, oldest_recv, oldest_sent, *_), (newest_now, newest_recv, newest_sent, *_) = \
history[0], history[-1]
elapsed = int(newest_now - oldest_now)
if elapsed == 0:
@ -351,6 +434,7 @@ def _calculate_bytes_per_second(history: List[Tuple]) -> Tuple[int, int, int]:
return bytes_recv_ps, bytes_sent_ps, elapsed
@limit_concurrent(1)
async def bandwidth_worker(count: int = None):
"""A background process which will gather historical data about all NIC bandwidth statistics."""
if not psutil:
@ -371,6 +455,17 @@ async def bandwidth_worker(count: int = None):
nic['historical'] = (nic['historical'] + [_get_nic_tick(name_), ])[-21:]
BANDWIDTH.update({name_: nic})
timestamp = now().timestamp()
for name_, disk in psutil.disk_io_counters(perdisk=True).items():
tic = timestamp, disk.read_bytes, disk.write_bytes
if bw := DRIVE_BANDWIDTH.get(name_):
bw['historical'] = (bw['historical'] + [tic, ])[-21:]
DRIVE_BANDWIDTH.update({name_: bw})
else:
DRIVE_BANDWIDTH.update({
name_: dict(historical=[tic, ]),
})
# Initialize the stats.
append_all_stats()
@ -394,6 +489,20 @@ async def bandwidth_worker(count: int = None):
speed=historical[-1][-1], # Use the most recent speed.
)
})
for name, stats in DRIVE_BANDWIDTH.items():
if 'historical' not in stats:
continue
historical = stats['historical']
bytes_read_ps, bytes_write_ps, elapsed = _calculate_bytes_per_second(historical)
DRIVE_BANDWIDTH.update({
name: dict(
historical=historical,
bytes_read_ps=bytes_read_ps,
bytes_write_ps=bytes_write_ps,
elapsed=elapsed,
)
})
@dataclass
@ -401,21 +510,26 @@ class Status:
cpu_info: CPUInfo
load: SystemLoad
drives: List[DriveInfo]
bandwidth: BandwidthInfo
bandwidth: NICBandwidthInfo
disk_bandwidth: DiskBandwidthInfo
memory_stats: MemoryStats
async def get_status() -> Status:
cpu_info, load, drives, bandwidth = await asyncio.gather(
cpu_info, load, drives, memory_stats, (nic_bandwidth, disk_bandwidth) = await asyncio.gather(
get_cpu_info(),
get_load(),
get_drives_info(),
get_memory_stats(),
get_bandwidth_info(),
)
return Status(
cpu_info=cpu_info,
load=load,
drives=drives,
bandwidth=bandwidth,
bandwidth=nic_bandwidth,
disk_bandwidth=disk_bandwidth,
memory_stats=memory_stats,
)
@ -428,6 +542,7 @@ if __name__ == '__main__':
task = asyncio.gather(
get_cpu_info(),
get_memory_stats(),
get_load(),
get_drives_info(),
get_bandwidth_info(),

View file

@ -4,12 +4,13 @@ import tempfile
import unittest
from datetime import date, datetime
from decimal import Decimal
from itertools import zip_longest
from unittest import mock
import pytest
from wrolpi.common import insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \
escape_file_name, match_paths_to_suffixes, truncate_object_bytes, check_media_directory
import wrolpi.vars
from wrolpi import common
from wrolpi.dates import set_timezone, now
from wrolpi.errors import InvalidTimezone
from wrolpi.test.common import build_test_directories
@ -63,51 +64,51 @@ def test_insert_parameter():
def func(foo, bar):
pass
results = insert_parameter(func, 'bar', 'bar', (1,), {})
results = common.insert_parameter(func, 'bar', 'bar', (1,), {})
assert results == ((1, 'bar'), {})
def func(foo, bar, baz):
pass
results = insert_parameter(func, 'bar', 'bar', (1, 2), {})
results = common.insert_parameter(func, 'bar', 'bar', (1, 2), {})
assert results == ((1, 'bar', 2), {})
def func(foo, baz, bar=None):
pass
results = insert_parameter(func, 'bar', 'bar', (1, 2), {})
results = common.insert_parameter(func, 'bar', 'bar', (1, 2), {})
assert results == ((1, 2, 'bar'), {})
def func(foo, baz, bar=None):
pass
results = insert_parameter(func, 'baz', 'baz', (1, 2), {})
results = common.insert_parameter(func, 'baz', 'baz', (1, 2), {})
assert results == ((1, 'baz', 2), {})
def func(foo, baz, qux=None, bar=None):
pass
results = insert_parameter(func, 'bar', 'bar', (1, 2, 3), {})
results = common.insert_parameter(func, 'bar', 'bar', (1, 2, 3), {})
assert results == ((1, 2, 3, 'bar'), {})
# bar is not defined as a parameter!
def func(foo):
pass
pytest.raises(TypeError, insert_parameter, func, 'bar', 'bar', (1,), {})
pytest.raises(TypeError, common.insert_parameter, func, 'bar', 'bar', (1,), {})
class TestCommon(unittest.TestCase):
def test_date_range(self):
# A single step results in the start.
result = date_range(date(1970, 1, 1), date(1970, 1, 2), 1)
result = common.date_range(date(1970, 1, 1), date(1970, 1, 2), 1)
assert result == [
date(1970, 1, 1),
]
# Many steps on a single day results in the same day.
result = date_range(date(1970, 1, 1), date(1970, 1, 1), 5)
result = common.date_range(date(1970, 1, 1), date(1970, 1, 1), 5)
assert result == [
date(1970, 1, 1),
date(1970, 1, 1),
@ -117,7 +118,7 @@ class TestCommon(unittest.TestCase):
]
# Many steps on a single datetime results in a range of times.
result = date_range(datetime(1970, 1, 1), datetime(1970, 1, 1, 23, 59, 59), 5)
result = common.date_range(datetime(1970, 1, 1), datetime(1970, 1, 1, 23, 59, 59), 5)
assert result == [
datetime(1970, 1, 1, 0, 0),
datetime(1970, 1, 1, 4, 47, 59, 800000),
@ -126,8 +127,8 @@ class TestCommon(unittest.TestCase):
datetime(1970, 1, 1, 19, 11, 59, 200000),
]
# date_range is not inclusive, like range().
result = date_range(date(1970, 1, 1), date(1970, 1, 5), 4)
# common.date_range is not inclusive, like range().
result = common.date_range(date(1970, 1, 1), date(1970, 1, 5), 4)
assert result == [
date(1970, 1, 1),
date(1970, 1, 2),
@ -136,7 +137,7 @@ class TestCommon(unittest.TestCase):
]
# Reversed dates are supported.
result = date_range(date(1970, 1, 5), date(1970, 1, 1), 4)
result = common.date_range(date(1970, 1, 5), date(1970, 1, 1), 4)
assert result == [
date(1970, 1, 5),
date(1970, 1, 4),
@ -145,7 +146,7 @@ class TestCommon(unittest.TestCase):
]
# Large date spans are supported.
result = date_range(date(1970, 1, 1), date(2020, 5, 1), 4)
result = common.date_range(date(1970, 1, 1), date(2020, 5, 1), 4)
assert result == [
date(1970, 1, 1),
date(1982, 8, 1),
@ -153,7 +154,7 @@ class TestCommon(unittest.TestCase):
date(2007, 10, 1),
]
result = date_range(datetime(1970, 1, 1, 0, 0, 0), datetime(1970, 1, 1, 10, 0), 8)
result = common.date_range(datetime(1970, 1, 1, 0, 0, 0), datetime(1970, 1, 1, 10, 0), 8)
assert result == [
datetime(1970, 1, 1, 0, 0),
datetime(1970, 1, 1, 1, 15),
@ -166,7 +167,7 @@ class TestCommon(unittest.TestCase):
]
# More steps than days
result = date_range(date(1970, 1, 1), date(1970, 1, 7), 10)
result = common.date_range(date(1970, 1, 1), date(1970, 1, 7), 10)
assert result == [
date(1970, 1, 1),
date(1970, 1, 1),
@ -215,7 +216,7 @@ class TestCommon(unittest.TestCase):
]
)
def test_api_param_limiter(i, expected):
limiter = api_param_limiter(100) # should never return an integer greater than 100.
limiter = common.api_param_limiter(100) # should never return an integer greater than 100.
assert limiter(i) == expected
@ -227,22 +228,22 @@ def test_chdir():
home = os.environ.get('HOME')
assert home
with chdir():
with common.chdir():
assert os.getcwd() != original
assert str(os.getcwd()).startswith('/tmp')
assert os.environ['HOME'] != os.getcwd()
# Replace $HOME
with chdir(with_home=True):
with common.chdir(with_home=True):
assert os.getcwd() != original
assert str(os.getcwd()).startswith('/tmp')
assert os.environ['HOME'] == os.getcwd()
with tempfile.TemporaryDirectory() as d:
# Without replacing $HOME
with chdir(pathlib.Path(d), with_home=True):
with common.chdir(pathlib.Path(d), with_home=True):
assert os.getcwd() == d
assert os.environ['HOME'] == os.getcwd()
with chdir(pathlib.Path(d)):
with common.chdir(pathlib.Path(d)):
assert os.getcwd() == d
assert os.environ['HOME'] != os.getcwd()
@ -271,7 +272,7 @@ def test_chdir():
])
])
def test_zig_zag(low, high, expected):
zagger = zig_zag(low, high)
zagger = common.zig_zag(low, high)
for i in expected:
result = next(zagger)
assert result == i
@ -297,7 +298,7 @@ def test_zig_zag(low, high, expected):
]
)
def test_escape_file_name(name, expected):
assert escape_file_name(name) == expected
assert common.escape_file_name(name) == expected
@pytest.mark.parametrize(
@ -319,37 +320,121 @@ def test_escape_file_name(name, expected):
def test_match_paths_to_suffixes(paths, suffix_groups, expected):
paths = [pathlib.Path(i) for i in paths]
expected = tuple(pathlib.Path(i) if i else None for i in expected)
assert (i := match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}'
assert (i := common.match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}'
def test_truncate_object_bytes():
"""
Objects can be truncated (lists will be shortened) so they will fit in tsvector columns.
"""
assert truncate_object_bytes(['foo'] * 10, 100) == ['foo'] * 5
assert truncate_object_bytes(['foo'] * 1_000, 100) == ['foo'] * 5
assert truncate_object_bytes(['foo'] * 1_000_000, 100) == ['foo'] * 5
assert truncate_object_bytes(['foo'] * 1_000_000, 200) == ['foo'] * 14
assert truncate_object_bytes([], 200) == []
assert common.truncate_object_bytes(['foo'] * 10, 100) == ['foo'] * 5
assert common.truncate_object_bytes(['foo'] * 1_000, 100) == ['foo'] * 5
assert common.truncate_object_bytes(['foo'] * 1_000_000, 100) == ['foo'] * 5
assert common.truncate_object_bytes(['foo'] * 1_000_000, 200) == ['foo'] * 14
assert common.truncate_object_bytes([], 200) == []
assert truncate_object_bytes(None, 100) is None
assert truncate_object_bytes('', 100) == ''
assert common.truncate_object_bytes(None, 100) is None
assert common.truncate_object_bytes('', 100) == ''
assert truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof'
assert truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo'
assert truncate_object_bytes('foo' * 100, 55) == 'foofo'
assert truncate_object_bytes('foo' * 100, 51) == 'f'
assert truncate_object_bytes('foo' * 100, 50) == ''
assert truncate_object_bytes('foo' * 100, 0) == ''
assert common.truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof'
assert common.truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo'
assert common.truncate_object_bytes('foo' * 100, 55) == 'foofo'
assert common.truncate_object_bytes('foo' * 100, 51) == 'f'
assert common.truncate_object_bytes('foo' * 100, 50) == ''
assert common.truncate_object_bytes('foo' * 100, 0) == ''
def test_check_media_directory(test_directory):
"""The directory provided by the test_directory fixture is a valid media directory."""
assert check_media_directory() is True
assert common.check_media_directory() is True
def test_bad_check_media_directory():
"""/dev/full is not a valid media directory, warnings are issued and the check fails."""
with mock.patch('wrolpi.common.get_media_directory') as mock_get_media_directory:
mock_get_media_directory.return_value = pathlib.Path('/dev/full')
assert check_media_directory() is False
assert common.check_media_directory() is False
def test_chunks_by_name(test_directory, make_files_structure):
"""`chunks_by_name` breaks a list of paths on the name change close to the size."""
with pytest.raises(ValueError):
assert list(common.chunks_by_name([], 0)) == [[]]
assert list(common.chunks_by_name([], 5)) == [[]]
assert list(common.chunks_by_name([1, 2, 3], 5)) == [[1, 2, 3]]
files = make_files_structure([
'foo.mp4', 'foo.txt', 'foo.png', 'foo.info.json',
'bar.mp4', 'bar.readability.txt', 'bar.jpeg', 'bar.info.json',
'baz.mp4', 'baz.txt', 'baz.jpg', 'baz.info.json',
'qux.mp4', 'qux.txt', 'qux.tif', 'qux.info.json',
])
def assert_chunks(size, files_, expected):
for chunk, expected_chunk in zip_longest(list(common.chunks_by_name(files_, size)), expected):
assert chunk == [test_directory / i for i in expected_chunk]
assert_chunks(8, files, [
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt',
'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt',
'foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'],
['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
])
assert_chunks(6, files, [
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt',
'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt',
'qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
])
assert_chunks(5, files, [
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt',
'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt',
'qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
])
assert_chunks(3, files, [
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt'],
['baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'],
['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
])
assert_chunks(1, files, [
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt'],
['baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'],
['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
])
files = make_files_structure(['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4'])
assert_chunks(1, files, [['1.mp4', '1.txt'], ['2.mp4', '2.png', '2.txt'], ['3.mp4']])
assert_chunks(3, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt'], ['3.mp4']])
assert_chunks(6, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt', '3.mp4']])
# List is not sorted because it is less than the size.
assert_chunks(20, files, [['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']])
@pytest.mark.parametrize(
'value,expected', [
('true', True),
('t', True),
('True', True),
('1', True),
('yes', True),
('no', False),
('0', False),
('False', False),
('false', False),
('f', False),
('other', False),
('trust', False),
('', False),
(None, False),
]
)
def test_truthy_arg(value, expected):
assert wrolpi.vars.truthy_arg(value) is expected, f'{value} != {expected}'

View file

@ -314,6 +314,7 @@ def test_get_status(test_client, test_session):
assert 'hotspot_status' in response.json and isinstance(response.json['hotspot_status'], str)
assert 'throttle_status' in response.json and isinstance(response.json['throttle_status'], str)
assert 'version' in response.json and isinstance(response.json['version'], str)
assert 'memory_stats' in response.json and isinstance(response.json['memory_stats'], dict)
def test_post_download(test_session, test_client, test_download_manager_config):

View file

@ -59,22 +59,26 @@ async def test_get_drives_info():
@pytest.mark.asyncio
async def test_get_bandwidth_info():
"""Bandwidth requires psutil"""
bandwidths = await status.get_bandwidth_info()
assert bandwidths == []
nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info()
assert nic_bandwidths == []
assert disk_bandwidths == []
await status.bandwidth_worker(2)
bandwidths = await status.get_bandwidth_info()
assert isinstance(bandwidths, list)
assert len(bandwidths) > 0
assert isinstance(bandwidths[0], status.BandwidthInfo)
nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info()
assert isinstance(nic_bandwidths, list)
assert len(nic_bandwidths) > 0
assert isinstance(nic_bandwidths[0], status.NICBandwidthInfo)
assert isinstance(disk_bandwidths, list)
assert len(disk_bandwidths) > 0
assert isinstance(disk_bandwidths[0], status.DiskBandwidthInfo)
with mock.patch('wrolpi.status.psutil.net_io_counters') as mock_net_io_counters:
# NO FALLBACK!
mock_net_io_counters.side_effect = Exception('testing no psutil')
await status.bandwidth_worker(1)
bandwidths = await status.get_bandwidth_info()
assert isinstance(bandwidths[0].bytes_recv, int)
assert isinstance(bandwidths[0].bytes_sent, int)
assert isinstance(bandwidths[0].elapsed, int)
assert isinstance(bandwidths[0].speed, int)
assert isinstance(bandwidths[0].name, str)
nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info()
assert isinstance(nic_bandwidths[0].bytes_recv, int)
assert isinstance(nic_bandwidths[0].bytes_sent, int)
assert isinstance(nic_bandwidths[0].elapsed, int)
assert isinstance(nic_bandwidths[0].speed, int)
assert isinstance(nic_bandwidths[0].name, str)

View file

@ -1,3 +1,4 @@
import multiprocessing
import os
import sys
from pathlib import Path
@ -15,6 +16,33 @@ DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else F
# tests are running
PYTEST = 'pytest' in sys.modules
def truthy_arg(value: str) -> bool:
"""Attempts to convert `value` from console arguments to a boolean value.
>>> truthy_arg('true')
True
>>> truthy_arg('True')
True
>>> truthy_arg('t')
True
>>> truthy_arg('1')
True
>>> truthy_arg('yes')
True
>>> truthy_arg('0')
False
>>> truthy_arg('false')
False
>>> truthy_arg('f')
False
>>> truthy_arg('no')
False
"""
v = str(value).lower()
return v == 'true' or v == 't' or v == '1' or v == 'yes'
# Get the media directory from the environment.
DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
@ -37,3 +65,12 @@ DB_PORT = int(os.environ.get('DB_PORT', 5432))
DB_NAME = os.environ.get('DB_NAME', 'wrolpi')
DB_USER = os.environ.get('DB_USER', 'wrolpi')
DB_PASSWORD = os.environ.get('DB_PASSWORD', 'wrolpi')
API_HOST = os.environ.get('API_HOST', '127.0.0.1')
API_PORT = os.environ.get('API_PORT', '8081')
API_WORKERS = os.environ.get('API_WORKERS', multiprocessing.cpu_count())
API_AUTO_RELOAD = truthy_arg(os.environ.get('API_AUTO_RELOAD', DOCKERIZED))
API_ACCESS_LOG = truthy_arg(os.environ.get('API_ACCESS_LOG', DOCKERIZED))
API_DEBUG = truthy_arg(os.environ.get('API_DEBUG', False))
FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100))

View file

@ -1,4 +1,4 @@
__version__ = '0.7.0-beta'
__version__ = '0.7.1-beta'
import subprocess