mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
Stopping renderd when reseting map db.
Skipping channel import during test unless explicitly requested. Deleting gis database during upgrade, if user agrees. Increasing API verbosity. Using status context to prevent fetching status in useHotspot. Disabling wifi toggle button when pending. Toggling hotspot when clicking nav hotspot icon. Displaying Hotspot status icon in nav menu. Importing channels/favorites after refreshing. Refresh tasks cancel when server shuts down. Bumping version for release speed. Refreshing files is 3x faster. Creating useRecurringTimeout hook which repeatedly calls a callback after a timeout. Changing intervals to timeout so fetches don't stack up while server is loaded. Checking that video file exists before fetching. Creating release candidate branch. Not indexing model associated files.
This commit is contained in:
parent
dbdcc7a57c
commit
6fa6be50fe
32 changed files with 878 additions and 358 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import React, {useContext, useEffect, useState} from "react";
|
||||
import {Card, Container, Image, Input, Pagination, Responsive, TableCell, TableRow} from 'semantic-ui-react';
|
||||
import {Card, Container, IconGroup, Image, Input, Pagination, Responsive, TableCell, TableRow} from 'semantic-ui-react';
|
||||
import {Link, NavLink} from "react-router-dom";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks";
|
||||
|
|
@ -734,6 +734,24 @@ export function DarkModeToggle() {
|
|||
</>
|
||||
}
|
||||
|
||||
export function HotspotStatusIcon() {
|
||||
const {on, setHotspot} = useHotspot();
|
||||
const toggleHotspot = (e) => {
|
||||
e.preventDefault();
|
||||
if (on != null) {
|
||||
setHotspot(!on);
|
||||
}
|
||||
}
|
||||
|
||||
return <a href='#' onClick={toggleHotspot}>
|
||||
<IconGroup size='large'>
|
||||
<Icon name='wifi' disabled={on !== true}/>
|
||||
{on === false && <Icon corner name='x'/>}
|
||||
{on === null && <Icon corner name='question'/>}
|
||||
</IconGroup>
|
||||
</a>
|
||||
}
|
||||
|
||||
export function useTitle(title) {
|
||||
const documentDefined = typeof document !== 'undefined';
|
||||
const originalTitle = React.useRef(documentDefined ? document.title : null);
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import {NavLink} from "react-router-dom";
|
|||
import {Dropdown, Menu, Responsive} from "semantic-ui-react";
|
||||
import {StatusContext, ThemeContext} from "../contexts/contexts";
|
||||
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
|
||||
import {DarkModeToggle} from "./Common";
|
||||
import {DarkModeToggle, HotspotStatusIcon} from "./Common";
|
||||
|
||||
const responsiveWidth = 500;
|
||||
|
||||
|
|
@ -63,6 +63,10 @@ function MenuLink(props) {
|
|||
}
|
||||
}
|
||||
|
||||
function NavIcon(props) {
|
||||
return <div style={{margin: '0.8em'}}>{props.children}</div>
|
||||
}
|
||||
|
||||
export function NavBar() {
|
||||
const {status} = useContext(StatusContext);
|
||||
const wrol_mode = status ? status['wrol_mode'] : null;
|
||||
|
|
@ -81,17 +85,15 @@ export function NavBar() {
|
|||
)
|
||||
})}
|
||||
<Responsive minWidth={responsiveWidth} as={Menu.Menu} position="right">
|
||||
<div style={{margin: '0.8em'}}>
|
||||
<DarkModeToggle/>
|
||||
</div>
|
||||
<NavIcon><HotspotStatusIcon/></NavIcon>
|
||||
<NavIcon><DarkModeToggle/></NavIcon>
|
||||
{rightLinks.map((link) => <MenuLink link={link} key={link.key}/>)}
|
||||
</Responsive>
|
||||
|
||||
{/*Show the menu items in a dropdown when on mobile*/}
|
||||
<Responsive as={Menu.Menu} maxWidth={responsiveWidth - 1} position='right'>
|
||||
<div style={{margin: '0.8em'}}>
|
||||
<DarkModeToggle/>
|
||||
</div>
|
||||
<NavIcon><HotspotStatusIcon/></NavIcon>
|
||||
<NavIcon><DarkModeToggle/></NavIcon>
|
||||
<Dropdown item icon="bars">
|
||||
<Dropdown.Menu>
|
||||
{collapsedLinks.map((link) =>
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ function Statistics() {
|
|||
}
|
||||
|
||||
const videoNames = [
|
||||
{key: 'videos', label: 'Downloaded Videos'},
|
||||
{key: 'videos', label: 'Videos'},
|
||||
{key: 'favorites', label: 'Favorite Videos'},
|
||||
{key: 'sum_size', label: 'Total Size'},
|
||||
{key: 'max_size', label: 'Largest Video'},
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import {Header, Loader, Progress, Statistic, StatisticGroup} from "../Theme";
|
||||
import React from "react";
|
||||
import {Header, Progress, Statistic, StatisticGroup} from "../Theme";
|
||||
import React, {useContext} from "react";
|
||||
import {humanBandwidth, humanFileSize, LoadStatistic} from "../Common";
|
||||
import {getStatus} from "../../api";
|
||||
import {Container, Divider} from "semantic-ui-react";
|
||||
import {Divider} from "semantic-ui-react";
|
||||
import {ProgressPlaceholder} from "../Placeholder";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
import {StatusContext} from "../../contexts/contexts";
|
||||
|
||||
function DriveInfo({used, size, percent, mount}) {
|
||||
let color;
|
||||
|
|
@ -102,75 +102,64 @@ export function CPUUsageProgress({value, label}) {
|
|||
return <Progress percent={value} progress color={color} label={label}/>
|
||||
}
|
||||
|
||||
export class Status extends React.Component {
|
||||
export function Status() {
|
||||
const {status} = useContext(StatusContext);
|
||||
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
status: null,
|
||||
}
|
||||
this.fetchStatus = this.fetchStatus.bind(this);
|
||||
let percent;
|
||||
let cores;
|
||||
let temperature;
|
||||
let high_temperature;
|
||||
let critical_temperature;
|
||||
let minute_1;
|
||||
let minute_5;
|
||||
let minute_15;
|
||||
let bandwidth;
|
||||
let drives = [];
|
||||
|
||||
if (status && status['cpu_info']) {
|
||||
const {cpu_info, load} = status;
|
||||
percent = cpu_info['percent'];
|
||||
cores = cpu_info['cores'];
|
||||
temperature = cpu_info['temperature'];
|
||||
high_temperature = cpu_info['high_temperature'];
|
||||
critical_temperature = cpu_info['critical_temperature'];
|
||||
|
||||
minute_1 = load['minute_1'];
|
||||
minute_5 = load['minute_5'];
|
||||
minute_15 = load['minute_15'];
|
||||
|
||||
drives = status['drives'];
|
||||
bandwidth = status['bandwidth'];
|
||||
}
|
||||
|
||||
async componentDidMount() {
|
||||
await this.fetchStatus();
|
||||
this.intervalId = setInterval(this.fetchStatus, 1000 * 5);
|
||||
}
|
||||
return <>
|
||||
<CPUUsageProgress value={percent} label='CPU Usage'/>
|
||||
|
||||
componentWillUnmount() {
|
||||
clearInterval(this.intervalId);
|
||||
}
|
||||
<CPUTemperatureStatistic
|
||||
value={temperature}
|
||||
high_temperature={high_temperature}
|
||||
critical_temperature={critical_temperature}
|
||||
/>
|
||||
<Statistic label='Cores' value={cores || '?'}/>
|
||||
|
||||
fetchStatus = async () => {
|
||||
try {
|
||||
let status = await getStatus();
|
||||
this.setState({status: status});
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
this.setState({status: null});
|
||||
}
|
||||
}
|
||||
<Divider/>
|
||||
|
||||
render() {
|
||||
if (this.state.status) {
|
||||
const {cpu_info, load, drives, bandwidth} = this.state.status;
|
||||
const {minute_1, minute_5, minute_15} = load;
|
||||
const {percent, cores, temperature, high_temperature, critical_temperature} = cpu_info;
|
||||
<StatisticGroup>
|
||||
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
|
||||
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
|
||||
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
|
||||
</StatisticGroup>
|
||||
|
||||
return <>
|
||||
<CPUUsageProgress value={percent} label='CPU Usage'/>
|
||||
<Divider/>
|
||||
|
||||
<CPUTemperatureStatistic
|
||||
value={temperature}
|
||||
high_temperature={high_temperature}
|
||||
critical_temperature={critical_temperature}
|
||||
/>
|
||||
<Statistic label='Cores' value={cores || '?'}/>
|
||||
<Header as='h1'>Bandwidth</Header>
|
||||
{bandwidth ?
|
||||
bandwidth.map(i => <BandwidthProgressGroup key={i['name']} bandwidth={i}/>) :
|
||||
<ProgressPlaceholder/>}
|
||||
|
||||
<Divider/>
|
||||
<Divider/>
|
||||
|
||||
<StatisticGroup>
|
||||
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
|
||||
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
|
||||
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
|
||||
</StatisticGroup>
|
||||
|
||||
<Divider/>
|
||||
|
||||
<Header as='h1'>Bandwidth</Header>
|
||||
{bandwidth ?
|
||||
bandwidth.map(i => <BandwidthProgressGroup key={i['name']} bandwidth={i}/>) :
|
||||
<ProgressPlaceholder/>}
|
||||
|
||||
<Divider/>
|
||||
|
||||
<Header as='h1'>Drive Usage</Header>
|
||||
{drives.map((drive) => <DriveInfo key={drive['mount']} {...drive}/>)}
|
||||
</>
|
||||
}
|
||||
|
||||
return <Container fluid style={{marginBottom: '5em'}}>
|
||||
<Loader active inline='centered' size='big'>Loading system status</Loader>
|
||||
</Container>
|
||||
}
|
||||
<Header as='h1'>Drive Usage</Header>
|
||||
{drives.map((drive) => <DriveInfo key={drive['mount']} {...drive}/>)}
|
||||
</>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import {useEffect, useState} from "react";
|
||||
import {useContext, useEffect, useRef, useState} from "react";
|
||||
import {
|
||||
favoriteVideo,
|
||||
fetchDomains,
|
||||
|
|
@ -27,6 +27,7 @@ import {
|
|||
import {createSearchParams, useSearchParams} from "react-router-dom";
|
||||
import {toast} from "react-semantic-toasts";
|
||||
import {enumerate, humanFileSize, secondsToFullDuration} from "../components/Common";
|
||||
import {StatusContext} from "../contexts/contexts";
|
||||
|
||||
const calculatePage = (offset, limit) => {
|
||||
return offset && limit ? Math.round((offset / limit) + 1) : 1;
|
||||
|
|
@ -36,6 +37,25 @@ const calculateTotalPages = (total, limit) => {
|
|||
return total && limit ? Math.round(total / limit) + 1 : 1;
|
||||
}
|
||||
|
||||
export const useRecurringTimeout = (callback, delay) => {
|
||||
const timer = useRef(null);
|
||||
|
||||
useEffect(() => {
|
||||
const repeat = async () => {
|
||||
// Recursively call the callback.
|
||||
await callback();
|
||||
timer.current = window.setTimeout(repeat, delay);
|
||||
}
|
||||
|
||||
// Call the repeating function instantly to fetch data fast.
|
||||
repeat();
|
||||
|
||||
// Clear the timeout on unload.
|
||||
return () => clearTimeout(timer.current);
|
||||
}, [delay])
|
||||
}
|
||||
|
||||
|
||||
export const useDomains = () => {
|
||||
const [domains, setDomains] = useState(null);
|
||||
const [total, setTotal] = useState(null);
|
||||
|
|
@ -441,34 +461,24 @@ export const useBrowseFiles = () => {
|
|||
|
||||
export const useHotspot = () => {
|
||||
const [on, setOn] = useState(null);
|
||||
|
||||
const fetchHotspotStatus = async () => {
|
||||
let status;
|
||||
try {
|
||||
status = await getHotspotStatus();
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
if (status === 'connected') {
|
||||
// Hotspot is on.
|
||||
setOn(true);
|
||||
} else if (status === 'disconnected' || status === 'unavailable') {
|
||||
// Hotspot can be turned on.
|
||||
setOn(false);
|
||||
} else {
|
||||
// Hotspot is not supported. API is probably running in a Docker container.
|
||||
setOn(null);
|
||||
}
|
||||
}
|
||||
const {status} = useContext(StatusContext);
|
||||
|
||||
useEffect(() => {
|
||||
fetchHotspotStatus();
|
||||
}, []);
|
||||
if (status && status['hotspot_status']) {
|
||||
const {hotspot_status} = status;
|
||||
if (hotspot_status === 'connected') {
|
||||
setOn(true);
|
||||
} else if (hotspot_status === 'disconnected' || hotspot_status === 'unavailable') {
|
||||
setOn(false);
|
||||
} else {
|
||||
setOn(null);
|
||||
}
|
||||
}
|
||||
}, [status]);
|
||||
|
||||
const localSetHotspot = async (on) => {
|
||||
setOn(null);
|
||||
await setHotspot(on);
|
||||
await fetchHotspotStatus();
|
||||
}
|
||||
|
||||
return {on, setOn, setHotspot: localSetHotspot};
|
||||
|
|
@ -488,13 +498,7 @@ export const useDownloads = () => {
|
|||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
fetchDownloads();
|
||||
const interval = setInterval(() => {
|
||||
fetchDownloads();
|
||||
}, 1000 * 3);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
useRecurringTimeout(fetchDownloads, 3 * 1000);
|
||||
|
||||
return {onceDownloads, recurringDownloads, fetchDownloads}
|
||||
}
|
||||
|
|
@ -616,12 +620,7 @@ export const useSettings = () => {
|
|||
export const useSettingsInterval = () => {
|
||||
const {settings, fetchSettings} = useSettings();
|
||||
|
||||
useEffect(() => {
|
||||
const interval = setInterval(() => {
|
||||
fetchSettings();
|
||||
}, 1000 * 3);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
useRecurringTimeout(fetchSettings(), 1000 * 3);
|
||||
|
||||
return {settings, fetchSettings};
|
||||
}
|
||||
|
|
@ -644,27 +643,13 @@ export const useStatus = () => {
|
|||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
fetchStatus();
|
||||
const interval = setInterval(() => {
|
||||
// console.log('fetch');
|
||||
fetchStatus();
|
||||
}, 1000 * 5);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
return {status, fetchStatus}
|
||||
}
|
||||
|
||||
export const useStatusInterval = () => {
|
||||
const {status, fetchStatus} = useStatus();
|
||||
|
||||
useEffect(() => {
|
||||
const interval = setInterval(() => {
|
||||
fetchStatus();
|
||||
}, 1000 * 3);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
useRecurringTimeout(fetchStatus, 1000 * 3);
|
||||
|
||||
return {status, fetchStatus};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ PartOf=wrolpi.target
|
|||
|
||||
[Service]
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
ExecStart=/usr/bin/python3 /opt/wrolpi/main.py api
|
||||
ExecStart=/usr/bin/python3 /opt/wrolpi/main.py -v api
|
||||
Group=wrolpi
|
||||
User=wrolpi
|
||||
WorkingDirectory=/opt/wrolpi/
|
||||
|
|
|
|||
5
main.py
5
main.py
|
|
@ -1,7 +1,6 @@
|
|||
#! /usr/bin/env python3
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytz
|
||||
|
|
@ -12,6 +11,7 @@ from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin
|
|||
from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, wrol_mode_enabled
|
||||
from wrolpi.dates import set_timezone
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.files.lib import cancel_refresh_tasks
|
||||
from wrolpi.root_api import api_app
|
||||
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
|
||||
from wrolpi.version import get_version_string
|
||||
|
|
@ -214,10 +214,11 @@ async def bandwidth_worker(app: Sanic):
|
|||
|
||||
@root_api.api_app.signal(Event.SERVER_SHUTDOWN_BEFORE)
|
||||
@limit_concurrent(1)
|
||||
def handle_server_shutdown(*args, **kwargs):
|
||||
async def handle_server_shutdown(*args, **kwargs):
|
||||
"""Stop downloads when server is shutting down."""
|
||||
if not PYTEST:
|
||||
download_manager.stop()
|
||||
await cancel_refresh_tasks()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import itertools
|
||||
import json
|
||||
import pathlib
|
||||
from abc import ABC
|
||||
|
|
@ -63,10 +62,10 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session):
|
|||
if any(lib.is_singlefile_file(i.path) for i in group):
|
||||
archive_groups[stem] = group
|
||||
|
||||
count = itertools.cycle(range(101))
|
||||
for stem, group in archive_groups.items():
|
||||
readability_file, singlefile_file, readability_json_file, readability_txt_file, screenshot_file = \
|
||||
match_archive_files(group)
|
||||
|
||||
archive: Archive = Archive.find_by_path(singlefile_file.path, session)
|
||||
if not archive:
|
||||
archive = Archive(singlefile_file=singlefile_file)
|
||||
|
|
@ -78,16 +77,16 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session):
|
|||
|
||||
if archive.readability_file:
|
||||
archive.readability_file.associated = True
|
||||
archive.readability_file.do_index()
|
||||
archive.readability_file.do_stats()
|
||||
if archive.readability_json_file:
|
||||
archive.readability_json_file.associated = True
|
||||
archive.readability_json_file.do_index()
|
||||
archive.readability_json_file.do_stats()
|
||||
if archive.readability_txt_file:
|
||||
archive.readability_txt_file.associated = True
|
||||
archive.readability_txt_file.do_index()
|
||||
archive.readability_txt_file.do_stats()
|
||||
if archive.screenshot_file:
|
||||
archive.screenshot_file.associated = True
|
||||
archive.screenshot_file.do_index()
|
||||
archive.screenshot_file.do_stats()
|
||||
|
||||
archive.singlefile_file.model = Archive.__tablename__
|
||||
|
||||
|
|
@ -97,10 +96,6 @@ def archive_modeler(groups: Dict[str, List[File]], session: Session):
|
|||
# Remove this group, it will not be processed by other modelers.
|
||||
del groups[stem]
|
||||
|
||||
if next(count) == 100:
|
||||
# Commit occasionally.
|
||||
session.commit()
|
||||
|
||||
|
||||
@register_after_refresh
|
||||
@limit_concurrent(1)
|
||||
|
|
@ -130,15 +125,14 @@ def archive_cleanup():
|
|||
with get_db_session(commit=True) as session:
|
||||
# Remove any Domains without any Archives.
|
||||
domain_ids = [i[0] for i in session.execute('SELECT DISTINCT domain_id FROM archive') if i[0]]
|
||||
deleted = 0
|
||||
for domain in session.query(Domain):
|
||||
if domain.id not in domain_ids:
|
||||
deleted += 1
|
||||
session.delete(domain)
|
||||
|
||||
|
||||
@register_indexer('text/html')
|
||||
class ArchiveIndexer(Indexer, ABC):
|
||||
"""Gathers index data from the files associated with an Archive."""
|
||||
|
||||
@staticmethod
|
||||
def get_title(file: File):
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from sqlalchemy.orm.collections import InstrumentedList
|
|||
from wrolpi.common import ModelHelper, Base, logger
|
||||
from wrolpi.dates import TZDateTime
|
||||
from wrolpi.errors import InvalidArchive
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
|
|
@ -78,6 +79,7 @@ class Archive(Base, ModelHelper):
|
|||
return self.archive_datetime > other.archive_datetime
|
||||
|
||||
def __json__(self):
|
||||
stem, _ = split_path_stem_and_suffix(self.singlefile_path) if self.singlefile_path else (None, None)
|
||||
d = dict(
|
||||
archive_datetime=self.archive_datetime,
|
||||
domain=self.domain.dict() if self.domain else None,
|
||||
|
|
@ -88,7 +90,7 @@ class Archive(Base, ModelHelper):
|
|||
readability_txt_path=self.readability_txt_path,
|
||||
screenshot_path=self.screenshot_path,
|
||||
singlefile_path=self.singlefile_path,
|
||||
stem=self.singlefile_path.stem if self.singlefile_path else None,
|
||||
stem=stem,
|
||||
title=self.title,
|
||||
url=self.url,
|
||||
)
|
||||
|
|
@ -172,7 +174,8 @@ class Archive(Base, ModelHelper):
|
|||
|
||||
readability_json_file = self.readability_json_file
|
||||
if not readability_json_file or not readability_json_file.path.is_file():
|
||||
raise InvalidArchive(f'Archive must have a readability json file: {self.singlefile_file.path}')
|
||||
logger.warning(f'Archive must have a readability json file: {self.singlefile_file.path}')
|
||||
return
|
||||
|
||||
try:
|
||||
with readability_json_file.path.open() as fh:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from modules.archive.lib import get_or_create_domain, get_new_archive_files, del
|
|||
from modules.archive.models import Archive, Domain
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.root_api import CustomJSONEncoder
|
||||
from wrolpi.test.common import skip_circleci
|
||||
|
|
@ -430,6 +431,43 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
|
|||
assert response.json['files'][0]['archive']['archive_datetime'], 'Archive has no datetime'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_archives_index(test_session, test_directory, test_client, make_files_structure):
|
||||
"""Archives are indexed using ArchiveIndexer and archive_modeler."""
|
||||
# The start of a typical singlefile html file.
|
||||
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
|
||||
Page saved with SingleFile
|
||||
url: https://example.com
|
||||
saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)
|
||||
--><head><meta charset="utf-8">'''
|
||||
|
||||
singlefile, *_ = make_files_structure({
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.png': None,
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents',
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.json':
|
||||
'{"url": "https://example.com", "title": "the title"}',
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
|
||||
})
|
||||
|
||||
await refresh_files()
|
||||
|
||||
archive: Archive = test_session.query(Archive).one()
|
||||
assert archive.singlefile_path == singlefile
|
||||
assert archive.singlefile_file.a_text == 'the title'
|
||||
assert archive.singlefile_file.d_text == 'article text contents'
|
||||
|
||||
# The associated files are not indexed.
|
||||
for name in ('screenshot_file', 'readability_file', 'readability_json_file', 'readability_txt_file'):
|
||||
file = getattr(archive, name)
|
||||
assert file.associated is True, 'The associated file is not marked'
|
||||
assert file.indexed is False
|
||||
# file.a_text is the file name.
|
||||
assert not file.b_text
|
||||
assert not file.c_text
|
||||
assert not file.d_text
|
||||
|
||||
|
||||
def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure):
|
||||
"""
|
||||
Invalid Archives should be removed during refresh.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import itertools
|
||||
import json
|
||||
import pathlib
|
||||
from abc import ABC
|
||||
|
|
@ -22,12 +21,10 @@ __all__ = ['video_modeler', 'VideoIndexer']
|
|||
|
||||
@register_modeler
|
||||
def video_modeler(groups: Dict[str, List[File]], session: Session):
|
||||
count = itertools.cycle(range(101))
|
||||
|
||||
# Only process groups that contain a video file.
|
||||
local_groups = groups.copy()
|
||||
videos = []
|
||||
for stem, group in local_groups.items():
|
||||
video_file = next((i for i in group if i.mimetype.split('/')[0] == 'video' and i.path.suffix != '.part'), None)
|
||||
video_file = next((i for i in group if i.mimetype.startswith('video/') and i.path.suffix != '.part'), None)
|
||||
if not video_file:
|
||||
# Not a video group.
|
||||
continue
|
||||
|
|
@ -40,28 +37,34 @@ def video_modeler(groups: Dict[str, List[File]], session: Session):
|
|||
|
||||
if poster_file:
|
||||
poster_file.associated = True
|
||||
poster_file.do_index()
|
||||
poster_file.do_stats()
|
||||
if caption_file:
|
||||
caption_file.associated = True
|
||||
caption_file.do_index()
|
||||
caption_file.do_stats()
|
||||
if info_json_file:
|
||||
info_json_file.associated = True
|
||||
info_json_file.do_index()
|
||||
info_json_file.do_stats()
|
||||
|
||||
video: Video = Video.upsert(video_file, session)
|
||||
video = session.query(Video).filter_by(video_file=video_file).one_or_none()
|
||||
if not video:
|
||||
video = Video(video_file=video_file)
|
||||
session.add(video)
|
||||
video.poster_file = poster_file
|
||||
video.caption_file = caption_file
|
||||
video.info_json_file = info_json_file
|
||||
video.size = video_file.path.stat().st_size
|
||||
video_file.model = Video.__tablename__
|
||||
|
||||
video.video_file.do_index()
|
||||
video.validate(session)
|
||||
videos.append(video)
|
||||
|
||||
# Remove this group, it will not be processed by other modelers.
|
||||
del groups[stem]
|
||||
if videos:
|
||||
session.flush(videos)
|
||||
for video in videos:
|
||||
video.video_file.do_index()
|
||||
video.validate(session)
|
||||
|
||||
if next(count) == 100:
|
||||
# Commit occasionally.
|
||||
session.commit()
|
||||
# Remove this group, it will not be processed by other modelers.
|
||||
del groups[video.video_path.stem]
|
||||
|
||||
|
||||
@register_after_refresh
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from modules.videos.channel import lib
|
|||
from modules.videos.lib import save_channels_config, import_channels_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.errors import UnknownChannel
|
||||
from wrolpi.files.lib import refresh_files
|
||||
|
||||
|
||||
@pytest.mark.parametrize('params', [
|
||||
|
|
@ -93,7 +94,8 @@ def test_channels_no_url(test_session, test_directory):
|
|||
import_channels_config()
|
||||
|
||||
|
||||
def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory):
|
||||
"""A Video will be assigned to a Channel by its info json first, then to its directory."""
|
||||
channel1 = channel_factory(source_id='channel1')
|
||||
channel2 = channel_factory(source_id='channel2')
|
||||
|
|
@ -110,20 +112,20 @@ def test_refresh_videos_finds_channel(test_session, test_client, channel_factory
|
|||
with mock.patch('wrolpi.common.after_refresh') as mock_after_refresh:
|
||||
# Channel can be found in `video_cleanup`, prevent that with mock.
|
||||
mock_after_refresh.side_effect = []
|
||||
test_client.post('/api/files/refresh')
|
||||
await refresh_files()
|
||||
|
||||
# Channel in info_json was found during validation.
|
||||
assert vid1.channel_id == channel1.id
|
||||
|
||||
# Refresh using cleanup, info json should be trusted.
|
||||
test_client.post('/api/files/refresh')
|
||||
await refresh_files()
|
||||
assert vid1.channel_id == channel1.id
|
||||
|
||||
# Remove the info json, the Video's Channel will be its directory.
|
||||
vid1.channel_id = None
|
||||
vid1.info_json_path.unlink()
|
||||
test_session.commit()
|
||||
test_client.post('/api/files/refresh')
|
||||
await refresh_files()
|
||||
test_session.refresh(vid1)
|
||||
vid1 = test_session.query(Video).one()
|
||||
assert vid1.channel_id == channel2.id
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
|
|||
from yt_dlp import YoutubeDL
|
||||
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link
|
||||
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, after_refresh, register_after_refresh
|
||||
from wrolpi.dates import from_timestamp, Seconds
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.files.models import File
|
||||
|
|
@ -375,8 +375,14 @@ def save_channels_config(session=None, preserve_favorites: bool = True):
|
|||
|
||||
|
||||
@before_startup
|
||||
@register_after_refresh
|
||||
def import_channels_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
if PYTEST and not TEST_CHANNELS_CONFIG:
|
||||
logger.warning(f'Not importing channels during this test. Use `test_channels_config` fixture if you would'
|
||||
f'like to call this.')
|
||||
return
|
||||
|
||||
logger.info('Importing videos config')
|
||||
try:
|
||||
config = get_channels_config()
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from wrolpi.dates import now, TZDateTime
|
|||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.errors import UnknownVideo
|
||||
from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem
|
||||
from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
|
|
@ -240,7 +240,7 @@ class Video(ModelHelper, Base):
|
|||
def __json__(self):
|
||||
from modules.videos.common import minimize_video_info_json
|
||||
info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None
|
||||
stem = self.video_path.stem if self.video_path else None
|
||||
stem, _ = split_path_stem_and_suffix(self.video_path) if self.video_file else (None, None)
|
||||
|
||||
d = dict(
|
||||
caption_path=self.caption_path,
|
||||
|
|
|
|||
|
|
@ -3,13 +3,41 @@ import shutil
|
|||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.videos.models import Video, Channel
|
||||
from modules.videos.video import lib as video_lib
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_videos_index(test_session, test_directory, video_factory):
|
||||
"""A video discovered during refresh has files associated with it. Only the video file is indexed."""
|
||||
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
|
||||
test_session.commit()
|
||||
|
||||
test_session.query(Video).delete()
|
||||
test_session.query(File).delete()
|
||||
test_session.commit()
|
||||
|
||||
for _ in range(1):
|
||||
# Refreshing twice does not change the results.
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(File).count() == 4
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_file.indexed is True, 'Video was not indexed'
|
||||
assert video.video_file.a_text, 'Video title was not indexed'
|
||||
assert video.video_file.d_text, 'Video captions were not indexed'
|
||||
|
||||
assert not video.poster_file.indexed
|
||||
assert not video.caption_file.indexed
|
||||
assert not video.info_json_file.indexed
|
||||
|
||||
|
||||
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
|
||||
subdir = test_directory / 'subdir'
|
||||
subdir.mkdir()
|
||||
|
|
|
|||
|
|
@ -70,14 +70,14 @@ def test_parse_video_file_name(file_name, expected):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_associated_files(test_session, test_directory, video_file, image_file, vtt_file1):
|
||||
async def test_associated_video_files(test_session, test_directory, video_file, image_file, vtt_file1):
|
||||
"""A Video file has files related to it. They should be attached to the Video record."""
|
||||
caption_file = video_file.with_suffix('.en.vtt')
|
||||
vtt_file1.rename(caption_file)
|
||||
poster_file = video_file.with_suffix('.jpeg')
|
||||
image_file.rename(poster_file)
|
||||
|
||||
await files_lib._refresh_all_files()
|
||||
await files_lib.refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
# All files were found.
|
||||
|
|
@ -93,7 +93,7 @@ async def test_associated_files(test_session, test_directory, video_file, image_
|
|||
assert video.video_file.size
|
||||
assert video.video_file.indexed
|
||||
|
||||
await files_lib._refresh_all_files()
|
||||
await files_lib.refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
# No new files were added.
|
||||
|
|
@ -113,7 +113,7 @@ async def test_unassociated_video(test_session, test_directory, video_file):
|
|||
video_file.rename(new_video_file_path)
|
||||
video_file = new_video_file_path
|
||||
|
||||
await files_lib._refresh_all_files()
|
||||
await files_lib.refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
video: Video = test_session.query(Video).one()
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
|
|||
caption = video.video_file.d_text
|
||||
video = video.video_file.__json__()
|
||||
video['video']['caption'] = caption
|
||||
previous_video = previous_video.video_file.__json__() if previous_video else None
|
||||
next_video = next_video.video_file.__json__() if next_video else None
|
||||
previous_video = previous_video.video_file.__json__() if previous_video and previous_video.video_file else None
|
||||
next_video = next_video.video_file.__json__() if next_video and next_video.video_file else None
|
||||
|
||||
return video, previous_video, next_video
|
||||
|
||||
|
|
|
|||
|
|
@ -22,12 +22,12 @@ fi
|
|||
|
||||
if [[ ${1} == *.osm.pbf ]]; then
|
||||
# Import a PBF file.
|
||||
osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
|
||||
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
|
||||
nice -n 18 osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
|
||||
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 3 \
|
||||
-S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}"
|
||||
elif [[ ${1} == *.dump ]]; then
|
||||
# Import a Postgresql dump.
|
||||
pg_restore -j4 --no-owner -d gis "${1}"
|
||||
nice -n 18 pg_restore -j3 --no-owner -d gis "${1}"
|
||||
else
|
||||
echo "Cannot import unknown map file"
|
||||
exit 5
|
||||
|
|
|
|||
|
|
@ -18,6 +18,21 @@ apt install -y libboost-all-dev git tar unzip wget bzip2 build-essential autocon
|
|||
/usr/bin/npm install -g carto
|
||||
/usr/bin/carto -v
|
||||
|
||||
function yes_or_no {
|
||||
while true; do
|
||||
read -p "$* [y/n]:" yn
|
||||
case $yn in
|
||||
[Yy]*) return 0 ;;
|
||||
[Nn]*) return 1 ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
sudo -u postgres psql -c '\l' | grep gis &&
|
||||
yes_or_no "Map database already exists. Replace it?" &&
|
||||
systemctl stop renderd &&
|
||||
sudo -u postgres dropdb gis
|
||||
|
||||
# Create `gis` database for map.
|
||||
sudo -u postgres psql -c '\l' | grep gis || sudo -u postgres createdb -E UTF8 -O wrolpi gis
|
||||
sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" || :
|
||||
|
|
@ -64,7 +79,7 @@ while [ "${COUNT}" -lt "${MAX_TRIES}" ]; do
|
|||
external_data=true
|
||||
[[ "${dc_import}" = false ]] &&
|
||||
# Run import in "create" mode so basic tables will be created.
|
||||
sudo -u wrolpi osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \
|
||||
sudo -u wrolpi nice -n 18 osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \
|
||||
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
|
||||
-S /opt/openstreetmap-carto/openstreetmap-carto.style /tmp/district-of-columbia-latest.osm.pbf &&
|
||||
dc_import=true
|
||||
|
|
|
|||
147
wrolpi/common.py
147
wrolpi/common.py
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import atexit
|
||||
import contextlib
|
||||
import inspect
|
||||
import logging
|
||||
|
|
@ -26,7 +27,7 @@ from sqlalchemy.ext.compiler import compiles
|
|||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.dates import DEFAULT_TIMEZONE
|
||||
from wrolpi.dates import DEFAULT_TIMEZONE, now
|
||||
from wrolpi.errors import WROLModeEnabled, NativeOnly
|
||||
from wrolpi.vars import PYTEST, MODULES_DIR, \
|
||||
DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
|
||||
|
|
@ -37,6 +38,58 @@ formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(messag
|
|||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
__all__ = [
|
||||
'Base',
|
||||
'ModelHelper',
|
||||
'get_model_by_table_name',
|
||||
'tsvector',
|
||||
'compile_tsvector',
|
||||
'sanitize_link',
|
||||
'ConfigFile',
|
||||
'get_config',
|
||||
'set_test_config',
|
||||
'wrol_mode_enabled',
|
||||
'wrol_mode_check',
|
||||
'enable_wrol_mode',
|
||||
'disable_wrol_mode',
|
||||
'insert_parameter',
|
||||
'iterify',
|
||||
'date_range',
|
||||
'remove_whitespace',
|
||||
'run_after',
|
||||
'set_test_media_directory',
|
||||
'get_media_directory',
|
||||
'check_media_directory',
|
||||
'get_absolute_media_path',
|
||||
'get_relative_to_media_directory',
|
||||
'get_files_and_directories',
|
||||
'minimize_dict',
|
||||
'make_media_directory',
|
||||
'extract_domain',
|
||||
'import_modules',
|
||||
'api_param_limiter',
|
||||
'partition',
|
||||
'chdir',
|
||||
'zig_zag',
|
||||
'walk',
|
||||
'escape_file_name',
|
||||
'native_only',
|
||||
'recursive_map',
|
||||
'any_extensions',
|
||||
'aiohttp_post',
|
||||
'register_modeler',
|
||||
'apply_modelers',
|
||||
'register_after_refresh',
|
||||
'apply_after_refresh',
|
||||
'match_paths_to_suffixes',
|
||||
'chunks',
|
||||
'chunks_by_name',
|
||||
'timer',
|
||||
'cum_timer',
|
||||
'limit_concurrent',
|
||||
'truncate_object_bytes'
|
||||
]
|
||||
|
||||
# Base is used for all SQLAlchemy models.
|
||||
Base = declarative_base()
|
||||
|
||||
|
|
@ -369,11 +422,6 @@ def date_range(start: dt_or_d, end: dt_or_d, steps: int) -> List[dt_or_d]:
|
|||
return [start + (delta * i) for i in range(steps)]
|
||||
|
||||
|
||||
def chunks(it, size):
|
||||
it = iter(it)
|
||||
return iter(lambda: tuple(islice(it, size)), ())
|
||||
|
||||
|
||||
WHITESPACE = re.compile(r'\s')
|
||||
|
||||
|
||||
|
|
@ -763,7 +811,7 @@ def apply_modelers(files, session: Session):
|
|||
for file in files:
|
||||
if not file.mimetype:
|
||||
file.mimetype = get_mimetype(file.path)
|
||||
stem, suffix = split_path_stem_and_suffix(file.path)
|
||||
stem, _ = split_path_stem_and_suffix(file.path)
|
||||
try:
|
||||
groups[stem].append(file)
|
||||
except KeyError:
|
||||
|
|
@ -771,7 +819,6 @@ def apply_modelers(files, session: Session):
|
|||
|
||||
for modeler in modelers:
|
||||
modeler(groups, session)
|
||||
session.commit()
|
||||
|
||||
for stem, group in groups.items():
|
||||
# Index any files that were not claimed by modelers.
|
||||
|
|
@ -784,6 +831,7 @@ after_refresh = []
|
|||
|
||||
def register_after_refresh(func: callable):
|
||||
after_refresh.append(func)
|
||||
return func
|
||||
|
||||
|
||||
def apply_after_refresh():
|
||||
|
|
@ -807,6 +855,56 @@ def match_paths_to_suffixes(paths: List[pathlib.Path], suffix_groups: List[Tuple
|
|||
yield match
|
||||
|
||||
|
||||
def chunks(it: Iterable, size: int):
|
||||
"""Split an iterable into iterables of the defined length."""
|
||||
it = iter(it)
|
||||
return iter(lambda: tuple(islice(it, size)), ())
|
||||
|
||||
|
||||
def chunks_by_name(it: List[Union[pathlib.Path, str]], size: int) -> Generator[List[pathlib.Path], None, None]:
|
||||
"""
|
||||
Attempt to split a list of paths near the defined size. Keep groups of files together when they share
|
||||
matching names.
|
||||
|
||||
>>> files = ['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']
|
||||
>>> chunks_by_name(files, 2)
|
||||
[['1.mp4', '1.txt'], ['2.mp4', '2.txt', '2.png'], ['3.mp4']]
|
||||
>>> chunks_by_name(files, 3)
|
||||
[['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png'], ['3.mp4']]
|
||||
"""
|
||||
if not isinstance(size, int) or size < 1:
|
||||
raise ValueError('size must be a positive integer')
|
||||
|
||||
if not it or len(it) < size:
|
||||
yield it
|
||||
return
|
||||
|
||||
if not isinstance(it, list):
|
||||
raise ValueError('iterable must a list')
|
||||
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
it = sorted(it.copy())
|
||||
index = size
|
||||
last_name = None
|
||||
while it:
|
||||
if index >= len(it):
|
||||
# Ran out of items, yield what is left.
|
||||
yield it
|
||||
return
|
||||
path = it[index]
|
||||
name, _ = split_path_stem_and_suffix(path)
|
||||
if last_name and name != last_name:
|
||||
# Found a break in the path names, yield and reset.
|
||||
chunk, it = it[:index], it[index:]
|
||||
index = size
|
||||
last_name = None
|
||||
yield chunk
|
||||
continue
|
||||
# Didn't find a name change, try again.
|
||||
last_name = name
|
||||
index += 1
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def timer(name):
|
||||
"""Prints out the time elapsed during the call of some block."""
|
||||
|
|
@ -817,6 +915,35 @@ def timer(name):
|
|||
logger.warning(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
|
||||
|
||||
|
||||
TIMERS = dict()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def cum_timer(name: str):
|
||||
"""Track time usage within this context's name. The cumulative calls to this context will be printed on exit."""
|
||||
before = now()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
total_seconds = (now() - before).total_seconds()
|
||||
existing_seconds, calls = TIMERS.get(name, (0, 0))
|
||||
TIMERS[name] = (existing_seconds + total_seconds, calls + 1)
|
||||
|
||||
|
||||
@atexit.register
|
||||
def print_timer():
|
||||
"""Print any cumulative timers that have been stored during exit."""
|
||||
if not TIMERS:
|
||||
return
|
||||
|
||||
total = sum(i[0] for i in TIMERS.values())
|
||||
for name, (seconds, calls) in sorted(TIMERS.items(), key=lambda i: i[1]):
|
||||
percent = int((seconds / total) * 100)
|
||||
seconds = round(seconds, 5)
|
||||
print(f'CUM_TIMER: {repr(name)} elapsed {seconds} cumulative seconds ({percent}%)'
|
||||
f' {calls} calls', file=sys.stderr)
|
||||
|
||||
|
||||
def limit_concurrent(limit: int):
|
||||
"""
|
||||
Wrapper that limits the amount of concurrently running functions.
|
||||
|
|
@ -852,10 +979,6 @@ def limit_concurrent(limit: int):
|
|||
return wrapper
|
||||
|
||||
|
||||
async def count_files(path: pathlib.Path) -> int:
|
||||
return len([i for i in walk(path) if i.is_file()])
|
||||
|
||||
|
||||
def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> List[str]:
|
||||
"""
|
||||
Shorten an object. This is useful when inserting something into a tsvector.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import logging
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import ContextManager, Tuple, List, Union, Type
|
||||
|
|
@ -41,7 +42,7 @@ postgres_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{d
|
|||
|
||||
# This engine is used for all normal tasks (except testing).
|
||||
db_args = get_db_args()
|
||||
connect_args = dict(application_name='wrolpi_api')
|
||||
connect_args = dict(application_name='wrolpi_api', connect_timeout=600)
|
||||
uri = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**db_args)
|
||||
engine = create_engine(uri, poolclass=NullPool, connect_args=connect_args)
|
||||
session_maker = sessionmaker(bind=engine)
|
||||
|
|
@ -75,6 +76,9 @@ def get_db_context() -> Tuple[create_engine, Session]:
|
|||
return local_engine, session
|
||||
|
||||
|
||||
# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_db_session(commit: bool = False) -> ContextManager[Session]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -103,6 +103,8 @@ class TextIndexer(Indexer, ABC):
|
|||
|
||||
Detects VTT (caption) files and forwards them on."""
|
||||
|
||||
_ignored_suffixes = ('.json', '.vtt', '.srt', '.csv')
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file):
|
||||
path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
|
|
@ -120,10 +122,7 @@ class TextIndexer(Indexer, ABC):
|
|||
|
||||
@classmethod
|
||||
def detect_special_indexer(cls, file):
|
||||
if file.path.name.endswith('.json'):
|
||||
# JSON files are detected as "text/plain" but we don't want them to be indexed.
|
||||
return DefaultIndexer
|
||||
if file.path.name.endswith('.vtt') or file.path.name.endswith('.srt'):
|
||||
# Don't process caption files.
|
||||
if any(file.path.name.endswith(i) for i in cls._ignored_suffixes):
|
||||
# Some text files aren't for humans.
|
||||
return DefaultIndexer
|
||||
return cls
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import glob
|
|||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
from asyncio import Task
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Union
|
||||
|
|
@ -13,12 +14,12 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
|
||||
get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name
|
||||
get_files_and_directories, apply_modelers, apply_after_refresh, get_model_by_table_name, chunks_by_name
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
||||
from wrolpi.errors import InvalidFile
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.vars import PYTEST
|
||||
from wrolpi.vars import PYTEST, FILE_REFRESH_CHUNK_SIZE
|
||||
|
||||
try:
|
||||
import magic
|
||||
|
|
@ -88,20 +89,6 @@ def delete_file(file: str):
|
|||
|
||||
FILE_NAME_REGEX = re.compile(r'[_ .]')
|
||||
|
||||
|
||||
def split_file_name_into_words(path: Path) -> List[str]:
|
||||
"""
|
||||
Split a file name into words.
|
||||
|
||||
>>> split_path_stem_and_suffix(Path('foo.mp4'))
|
||||
['foo']
|
||||
>>> split_path_stem_and_suffix(Path('foo bar.mp4'))
|
||||
['foo', 'bar']
|
||||
"""
|
||||
words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)]
|
||||
return words
|
||||
|
||||
|
||||
FILE_BIN = which('file', '/usr/bin/file')
|
||||
|
||||
|
||||
|
|
@ -144,55 +131,88 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str]) -> Tuple[str, str
|
|||
return path.name, ''
|
||||
|
||||
|
||||
refresh_logger = logger.getChild('refresh')
|
||||
|
||||
|
||||
@wrol_mode_check
|
||||
def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime):
|
||||
async def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.datetime):
|
||||
"""Upsert, index, and model all provided paths."""
|
||||
# Update the idempotency if any files already in the DB.
|
||||
if not paths:
|
||||
return
|
||||
|
||||
paths = set(paths)
|
||||
for idx, chunk in enumerate(map(set, chunks_by_name(paths, FILE_REFRESH_CHUNK_SIZE))):
|
||||
with get_db_session(commit=True) as session:
|
||||
existing_files = list(session.query(File).filter(File.path.in_(chunk)))
|
||||
existing_paths = {i.path for i in existing_files}
|
||||
# Add any new files into the database.
|
||||
new_files = []
|
||||
if new_paths := (chunk - existing_paths):
|
||||
new_files = [File(path=i, idempotency=idempotency, mimetype=get_mimetype(i)) for i in new_paths]
|
||||
session.add_all(new_files)
|
||||
session.flush(new_files)
|
||||
# Apply models to all files. Many files will have no model.
|
||||
all_files = existing_files + new_files
|
||||
apply_modelers(all_files, session)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
existing_files = list(session.query(File).filter(File.path.in_(paths)))
|
||||
for existing_file in existing_files:
|
||||
existing_file.idempotency = idempotency
|
||||
session.commit()
|
||||
# Update idempotency for existing files after modelers. Otherwise SQLAlchemy forgets.
|
||||
for existing_file in existing_files:
|
||||
existing_file.idempotency = idempotency
|
||||
|
||||
existing_paths = {i.path for i in existing_files}
|
||||
# Add any new files into the database.
|
||||
new_files = []
|
||||
if new_paths := (paths - existing_paths):
|
||||
new_files = [File(path=i, idempotency=idempotency) for i in new_paths]
|
||||
session.add_all(new_files)
|
||||
# Commit before modeling. Just in case modeling fails.
|
||||
session.commit()
|
||||
if idx > 0:
|
||||
parent = all_files[0].path.parent
|
||||
refresh_logger.debug(f'Committed chunk of {len(chunk)} files in {parent}')
|
||||
|
||||
# Apply models to all files. Many files will have no model.
|
||||
apply_modelers(existing_files + new_files, session)
|
||||
# Sleep between chunks to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
refresh_logger = logger.getChild('refresh')
|
||||
|
||||
|
||||
async def _refresh_directory_files_recursively(directory, idempotency):
|
||||
"""
|
||||
Refresh all files in a directory, recursively refresh all files in subdirectories.
|
||||
"""
|
||||
async def _refresh_directory_files_recursively(directory: pathlib.Path, idempotency: datetime):
|
||||
"""Refresh all files in a directory, recursively refresh all files in subdirectories."""
|
||||
directories = [directory, ]
|
||||
while directories:
|
||||
directory = directories.pop(0)
|
||||
try:
|
||||
files, new_directories = get_files_and_directories(directory)
|
||||
refresh_logger.debug(f'Refreshing {len(files)} files in {directory}')
|
||||
directories.extend(new_directories)
|
||||
_refresh_files_list(files, idempotency)
|
||||
if files:
|
||||
refresh_logger.info(f'Refreshing {len(files)} files in {directory}')
|
||||
await _refresh_files_list(files, idempotency)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to refresh files in {directory}', exc_info=e)
|
||||
refresh_logger.error(f'Failed to refresh files in {directory}', exc_info=e)
|
||||
except asyncio.CancelledError:
|
||||
refresh_logger.error(f'Refresh canceled during {directory}')
|
||||
raise
|
||||
|
||||
|
||||
REFRESH_TASKS: List[Task] = []
|
||||
|
||||
|
||||
async def cancel_refresh_tasks():
|
||||
"""Cancel all refresh tasks, if any."""
|
||||
if REFRESH_TASKS:
|
||||
refresh_logger.warning(f'Canceling {len(REFRESH_TASKS)} refreshes')
|
||||
for task in REFRESH_TASKS:
|
||||
task.cancel()
|
||||
await asyncio.gather(*REFRESH_TASKS)
|
||||
|
||||
|
||||
def cancelable_wrapper(func: callable):
|
||||
@wraps(func)
|
||||
async def wrapped(*args, **kwargs):
|
||||
if PYTEST:
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
task = asyncio.create_task(func(*args, **kwargs))
|
||||
REFRESH_TASKS.append(task)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
@limit_concurrent(1) # Only one refresh at a time.
|
||||
async def _refresh_all_files():
|
||||
@wrol_mode_check
|
||||
@cancelable_wrapper
|
||||
async def refresh_files():
|
||||
"""Find, model, and index all files in the media directory."""
|
||||
refresh_logger.warning('Refreshing all files')
|
||||
|
||||
|
|
@ -217,18 +237,9 @@ async def _refresh_all_files():
|
|||
refresh_logger.warning('Done refreshing Files')
|
||||
|
||||
|
||||
@wraps(_refresh_all_files)
|
||||
@wrol_mode_check
|
||||
async def refresh_files():
|
||||
"""Schedule a refresh task if not testing. If testing, do a synchronous refresh."""
|
||||
if PYTEST:
|
||||
return await _refresh_all_files()
|
||||
|
||||
asyncio.create_task(_refresh_all_files())
|
||||
|
||||
|
||||
@limit_concurrent(1)
|
||||
@wrol_mode_check
|
||||
@cancelable_wrapper
|
||||
async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str]):
|
||||
"""Upsert and index all files within a directory (recursively).
|
||||
|
||||
|
|
|
|||
|
|
@ -91,18 +91,7 @@ class File(ModelHelper, Base):
|
|||
|
||||
def do_index(self, force_index: bool = False):
|
||||
"""Gather any missing information about this file. Index the contents of this file using an Indexer."""
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
|
||||
if not self.mimetype:
|
||||
from wrolpi.files.lib import get_mimetype
|
||||
self.mimetype = get_mimetype(self.path)
|
||||
|
||||
self.title = self.title or self.path.name
|
||||
_, self.suffix = split_path_stem_and_suffix(self.path)
|
||||
|
||||
stat = self.path.stat()
|
||||
self.size = stat.st_size
|
||||
self.modification_datetime = from_timestamp(stat.st_mtime)
|
||||
self.do_stats()
|
||||
|
||||
try:
|
||||
if force_index or self.indexed is not True:
|
||||
|
|
@ -119,6 +108,22 @@ class File(ModelHelper, Base):
|
|||
except Exception as e:
|
||||
logger.error(f'Failed to index {self.path}', exc_info=e)
|
||||
|
||||
def do_stats(self):
|
||||
"""Assign the mimetype, title, size, modification_time of this file."""
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
if not self.mimetype:
|
||||
from wrolpi.files.lib import get_mimetype
|
||||
self.mimetype = get_mimetype(self.path)
|
||||
self.title = self.title or self.path.name
|
||||
stat = self.path.stat()
|
||||
self.size = stat.st_size
|
||||
self.modification_datetime = from_timestamp(stat.st_mtime)
|
||||
name, self.suffix = split_path_stem_and_suffix(self.path)
|
||||
|
||||
# Use title as the a_text temporarily, this may be overwritten by indexer.
|
||||
# TODO this is very slow. What can be done instead?
|
||||
self.a_text = self.a_text or name
|
||||
|
||||
@validates('path')
|
||||
def validate_path(self, key, value):
|
||||
value = pathlib.Path(value) if not isinstance(value, pathlib.Path) else value
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import asyncio
|
||||
import json
|
||||
import shutil
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
|
|
@ -140,21 +143,6 @@ def test_delete_file(make_files_structure, test_directory):
|
|||
assert (test_directory / 'baz').is_dir()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'path,expected',
|
||||
[
|
||||
('foo.mp4', ['foo']),
|
||||
('foo bar.txt', ['foo', 'bar']),
|
||||
('foo_bar.mp4', ['foo', 'bar']),
|
||||
('foo-bar', ['foo-bar']),
|
||||
('123 foo bar', ['123', 'foo', 'bar']),
|
||||
('123foo bar', ['123foo', 'bar']),
|
||||
]
|
||||
)
|
||||
def test_split_file_name_into_words(path, expected):
|
||||
assert lib.split_file_name_into_words(Path(path)) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'path,expected',
|
||||
[
|
||||
|
|
@ -197,6 +185,42 @@ async def test_refresh_files(test_session, make_files_structure, test_directory)
|
|||
assert_files(test_session, [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_many_files(test_session, make_files_structure):
|
||||
count = 1_000
|
||||
make_files_structure([f'{uuid4()}.txt' for _ in range(count)])
|
||||
await lib.refresh_files()
|
||||
assert test_session.query(File).count() == count
|
||||
|
||||
await lib.refresh_files()
|
||||
assert test_session.query(File).count() == count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_cancel(test_session, make_files_structure, test_directory):
|
||||
"""Refresh tasks can be canceled."""
|
||||
# Creat a lot of files so the refresh will take too long.
|
||||
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
|
||||
|
||||
async def assert_cancel(task_):
|
||||
# Time the time it takes to cancel.
|
||||
before = datetime.now()
|
||||
# Sleep so the refresh task has time to run.
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Cancel the refresh (it will be sleeping soon).
|
||||
task_.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task_
|
||||
assert (datetime.now() - before).total_seconds() < 0.5, 'Task took too long. Was the refresh canceled?'
|
||||
|
||||
task = asyncio.create_task(lib.refresh_files())
|
||||
await assert_cancel(task)
|
||||
|
||||
task = asyncio.create_task(lib.refresh_directory_files_recursively(test_directory))
|
||||
await assert_cancel(task)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_files_in_directory(test_session, make_files_structure, test_directory):
|
||||
"""A subdirectory can be refreshed, files above it can be ignored."""
|
||||
|
|
@ -276,7 +300,7 @@ async def test_files_indexer(test_session, make_files_structure):
|
|||
# Enable slow feature for testing.
|
||||
# TODO can this be sped up to always be included?
|
||||
with mock.patch('modules.videos.EXTRACT_SUBTITLES', True):
|
||||
await lib._refresh_all_files()
|
||||
await lib.refresh_files()
|
||||
|
||||
bzip_file, gzip_file, text_file, zip_file, image_file, unknown_file, info_json_file, video_file \
|
||||
= test_session.query(File).order_by(File.path)
|
||||
|
|
@ -330,7 +354,7 @@ async def test_large_text_indexer(test_session, make_files_structure):
|
|||
large, = make_files_structure({
|
||||
'large_file.txt': 'foo ' * 1_000_000,
|
||||
})
|
||||
await lib._refresh_all_files()
|
||||
await lib.refresh_files()
|
||||
assert test_session.query(File).count() == 1
|
||||
|
||||
assert large.is_file() and large.stat().st_size == 4_000_000
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import json
|
||||
import multiprocessing
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, date
|
||||
from decimal import Decimal
|
||||
|
|
@ -24,14 +24,14 @@ from wrolpi.dates import set_timezone
|
|||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.errors import WROLModeEnabled, InvalidTimezone, API_ERRORS, APIError, ValidationError, HotspotError
|
||||
from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse
|
||||
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
|
||||
truthy_arg
|
||||
from wrolpi.version import __version__
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
api_app = Sanic(name='api_app')
|
||||
|
||||
DEFAULT_HOST, DEFAULT_PORT = '127.0.0.1', '8081'
|
||||
|
||||
api_bp = Blueprint('RootAPI', url_prefix='/api')
|
||||
|
||||
BLUEPRINTS = [api_bp, ]
|
||||
|
|
@ -50,26 +50,47 @@ def add_blueprint(bp: Union[Blueprint, BlueprintGroup]):
|
|||
BLUEPRINTS.append(bp)
|
||||
|
||||
|
||||
def run_webserver(host: str, port: int, workers: int = 8):
|
||||
def run_webserver(
|
||||
host: str = API_HOST,
|
||||
port: int = API_PORT,
|
||||
workers: int = API_WORKERS,
|
||||
api_debug: bool = API_DEBUG,
|
||||
access_log: bool = API_ACCESS_LOG,
|
||||
):
|
||||
# Attach all blueprints after they have been defined.
|
||||
for bp in BLUEPRINTS:
|
||||
api_app.blueprint(bp)
|
||||
|
||||
kwargs = dict(host=host, port=port, workers=workers, debug=False, access_log=False, auto_reload=True)
|
||||
logger.debug(f'Running Sanic {sanic_version} with kwargs {kwargs}')
|
||||
kwargs = dict(
|
||||
host=host,
|
||||
port=port,
|
||||
workers=workers,
|
||||
debug=api_debug,
|
||||
access_log=access_log,
|
||||
auto_reload=DOCKERIZED,
|
||||
)
|
||||
logger.warning(f'Running Sanic {sanic_version} with kwargs {kwargs}')
|
||||
return api_app.run(**kwargs)
|
||||
|
||||
|
||||
def init_parser(parser):
|
||||
# Called by WROLPI's main() function
|
||||
parser.add_argument('-H', '--host', default=DEFAULT_HOST, help='What network interface to connect webserver')
|
||||
parser.add_argument('-p', '--port', default=DEFAULT_PORT, type=int, help='What port to connect webserver')
|
||||
parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(), type=int,
|
||||
help='How many web workers to run')
|
||||
parser.add_argument('-H', '--host', default=API_HOST, help='What network interface to connect webserver')
|
||||
parser.add_argument('-p', '--port', default=API_PORT, type=int, help='What port to connect webserver')
|
||||
parser.add_argument('-w', '--workers', default=API_WORKERS, type=int, help='How many web workers to run')
|
||||
parser.add_argument('--access-log', default=API_ACCESS_LOG, type=truthy_arg, help='Enable Sanic access log')
|
||||
parser.add_argument('--api-debug', default=API_DEBUG, type=truthy_arg, help='Enable Sanic debug log')
|
||||
parser.add_argument('--api-auto-reload', default=API_AUTO_RELOAD, type=truthy_arg, help='Enable Sanic auto reload')
|
||||
|
||||
|
||||
def main(args):
|
||||
return run_webserver(args.host, args.port, args.workers)
|
||||
return run_webserver(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
workers=args.workers,
|
||||
api_debug=args.api_debug,
|
||||
access_log=args.access_log,
|
||||
)
|
||||
|
||||
|
||||
index_html = '''
|
||||
|
|
@ -313,14 +334,21 @@ async def throttle_off(_: Request):
|
|||
@openapi.description('Get the status of CPU/load/etc.')
|
||||
async def get_status(_: Request):
|
||||
s = await status.get_status()
|
||||
downloads = download_manager.get_summary()
|
||||
try:
|
||||
downloads = download_manager.get_summary()
|
||||
except Exception as e:
|
||||
logger.error('Unable to get download status', exc_info=e)
|
||||
downloads = dict()
|
||||
|
||||
ret = dict(
|
||||
bandwidth=s.bandwidth,
|
||||
cpu_info=s.cpu_info,
|
||||
disk_bandwidth=s.disk_bandwidth,
|
||||
downloads=downloads,
|
||||
drives=s.drives,
|
||||
hotspot_status=admin.hotspot_status().name,
|
||||
load=s.load,
|
||||
memory_stats=s.memory_stats,
|
||||
throttle_status=admin.throttle_status().name,
|
||||
version=__version__,
|
||||
wrol_mode=wrol_mode_enabled(),
|
||||
|
|
|
|||
149
wrolpi/status.py
149
wrolpi/status.py
|
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
import asyncio
|
||||
import multiprocessing
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -9,7 +10,7 @@ from pathlib import Path
|
|||
from typing import List, Optional, Tuple
|
||||
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import logger
|
||||
from wrolpi.common import logger, limit_concurrent
|
||||
from wrolpi.dates import now
|
||||
|
||||
try:
|
||||
|
|
@ -41,9 +42,9 @@ class SystemLoad:
|
|||
|
||||
def __json__(self):
|
||||
return dict(
|
||||
minute_1=str(self.minute_1),
|
||||
minute_5=str(self.minute_5),
|
||||
minute_15=str(self.minute_15),
|
||||
minute_1=str(round(self.minute_1, 2)),
|
||||
minute_5=str(round(self.minute_5, 2)),
|
||||
minute_15=str(round(self.minute_15, 2)),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -179,6 +180,60 @@ async def get_cpu_info() -> CPUInfo:
|
|||
return info
|
||||
|
||||
|
||||
MEMORY_TOTAL = re.compile(r'^MemTotal:\s+(\d+)\s+kB$', re.MULTILINE)
|
||||
MEMORY_FREE = re.compile(r'^MemFree:\s+(\d+)\s+kB$', re.MULTILINE)
|
||||
MEMORY_CACHED = re.compile(r'^Cached:\s+(\d+)\s+kB$', re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryStats:
|
||||
total: int = None
|
||||
used: int = None
|
||||
free: int = None
|
||||
cached: int = None
|
||||
|
||||
def __json__(self):
|
||||
return dict(
|
||||
total=self.total,
|
||||
used=self.used,
|
||||
free=self.free,
|
||||
cached=self.cached,
|
||||
)
|
||||
|
||||
|
||||
def get_memory_stats_psutil():
|
||||
mem = psutil.virtual_memory()
|
||||
mem = MemoryStats(
|
||||
total=mem.total,
|
||||
used=mem.used,
|
||||
free=mem.free,
|
||||
cached=mem.cached,
|
||||
)
|
||||
return mem
|
||||
|
||||
|
||||
async def get_memory_stats():
|
||||
try:
|
||||
return get_memory_stats_psutil()
|
||||
except Exception as e:
|
||||
warn_once(e)
|
||||
|
||||
meminfo = pathlib.Path('/proc/meminfo')
|
||||
if meminfo.is_file() and (contents := meminfo.read_text()):
|
||||
total = int(i[0]) if (i := MEMORY_TOTAL.findall(contents)) else None
|
||||
free = int(i[0]) if (i := MEMORY_FREE.findall(contents)) else None
|
||||
cached = int(i[0]) if (i := MEMORY_CACHED.findall(contents)) else None
|
||||
mem = MemoryStats(
|
||||
total=int(total * 1024) if total else None,
|
||||
free=int(free * 1024) if free else None,
|
||||
cached=int(cached * 1024) if cached else None,
|
||||
used=int(total - free) if total and free else None,
|
||||
)
|
||||
return mem
|
||||
|
||||
return MemoryStats()
|
||||
|
||||
|
||||
@dataclass
|
||||
class DriveInfo:
|
||||
mount: str = None
|
||||
|
|
@ -263,7 +318,7 @@ async def get_drives_info() -> List[DriveInfo]:
|
|||
|
||||
|
||||
@dataclass
|
||||
class BandwidthInfo:
|
||||
class NICBandwidthInfo:
|
||||
bytes_recv: int = None
|
||||
bytes_sent: int = None
|
||||
elapsed: int = None
|
||||
|
|
@ -288,6 +343,8 @@ IGNORED_NIC_NAMES = {
|
|||
'br-',
|
||||
}
|
||||
|
||||
BANDWIDTH = multiprocessing.Manager().dict()
|
||||
|
||||
|
||||
def get_nic_names() -> List[str]:
|
||||
"""Finds all non-virtual and non-docker network interface names."""
|
||||
|
|
@ -299,31 +356,57 @@ def get_nic_names() -> List[str]:
|
|||
return names
|
||||
|
||||
|
||||
BANDWIDTH = multiprocessing.Manager().dict()
|
||||
@dataclass
|
||||
class DiskBandwidthInfo:
|
||||
bytes_read: int = None
|
||||
bytes_write: int = None
|
||||
elapsed: int = None
|
||||
name: str = None
|
||||
|
||||
def __json__(self):
|
||||
return dict(
|
||||
bytes_read=self.bytes_read,
|
||||
bytes_write=self.bytes_write,
|
||||
elapsed=self.elapsed,
|
||||
name=self.name,
|
||||
)
|
||||
|
||||
|
||||
async def get_bandwidth_info() -> List[BandwidthInfo]:
|
||||
"""
|
||||
Get all bandwidth information for all NICs.
|
||||
"""
|
||||
infos = []
|
||||
DRIVE_BANDWIDTH = multiprocessing.Manager().dict()
|
||||
|
||||
|
||||
async def get_bandwidth_info() -> Tuple[List[NICBandwidthInfo], List[DiskBandwidthInfo]]:
|
||||
"""Get all bandwidth information for all NICs."""
|
||||
nics_info = []
|
||||
disks_info = []
|
||||
try:
|
||||
for name in sorted(BANDWIDTH.keys()):
|
||||
nic = BANDWIDTH[name]
|
||||
if 'bytes_recv_ps' not in nic:
|
||||
# Not stats collected yet.
|
||||
continue
|
||||
infos.append(BandwidthInfo(
|
||||
nics_info.append(NICBandwidthInfo(
|
||||
bytes_recv=nic['bytes_recv_ps'],
|
||||
bytes_sent=nic['bytes_sent_ps'],
|
||||
elapsed=nic['elapsed'],
|
||||
name=name,
|
||||
speed=nic['speed'],
|
||||
))
|
||||
for name in sorted(DRIVE_BANDWIDTH.keys()):
|
||||
disk = DRIVE_BANDWIDTH[name]
|
||||
if 'bytes_read_ps' not in disk:
|
||||
# No status collected yet
|
||||
continue
|
||||
disks_info.append(DiskBandwidthInfo(
|
||||
bytes_read=disk['bytes_read_ps'],
|
||||
bytes_write=disk['bytes_write_ps'],
|
||||
elapsed=disk['elapsed'],
|
||||
name=name,
|
||||
))
|
||||
except Exception as e:
|
||||
warn_once(e)
|
||||
|
||||
return infos
|
||||
return nics_info, disks_info
|
||||
|
||||
|
||||
def _get_nic_tick(name_):
|
||||
|
|
@ -341,7 +424,7 @@ def _get_nic_tick(name_):
|
|||
|
||||
def _calculate_bytes_per_second(history: List[Tuple]) -> Tuple[int, int, int]:
|
||||
"""Calculate the bytes-per-second between the oldest and newest tick."""
|
||||
(oldest_now, oldest_recv, oldest_sent, _), (newest_now, newest_recv, newest_sent, _) = \
|
||||
(oldest_now, oldest_recv, oldest_sent, *_), (newest_now, newest_recv, newest_sent, *_) = \
|
||||
history[0], history[-1]
|
||||
elapsed = int(newest_now - oldest_now)
|
||||
if elapsed == 0:
|
||||
|
|
@ -351,6 +434,7 @@ def _calculate_bytes_per_second(history: List[Tuple]) -> Tuple[int, int, int]:
|
|||
return bytes_recv_ps, bytes_sent_ps, elapsed
|
||||
|
||||
|
||||
@limit_concurrent(1)
|
||||
async def bandwidth_worker(count: int = None):
|
||||
"""A background process which will gather historical data about all NIC bandwidth statistics."""
|
||||
if not psutil:
|
||||
|
|
@ -371,6 +455,17 @@ async def bandwidth_worker(count: int = None):
|
|||
nic['historical'] = (nic['historical'] + [_get_nic_tick(name_), ])[-21:]
|
||||
BANDWIDTH.update({name_: nic})
|
||||
|
||||
timestamp = now().timestamp()
|
||||
for name_, disk in psutil.disk_io_counters(perdisk=True).items():
|
||||
tic = timestamp, disk.read_bytes, disk.write_bytes
|
||||
if bw := DRIVE_BANDWIDTH.get(name_):
|
||||
bw['historical'] = (bw['historical'] + [tic, ])[-21:]
|
||||
DRIVE_BANDWIDTH.update({name_: bw})
|
||||
else:
|
||||
DRIVE_BANDWIDTH.update({
|
||||
name_: dict(historical=[tic, ]),
|
||||
})
|
||||
|
||||
# Initialize the stats.
|
||||
append_all_stats()
|
||||
|
||||
|
|
@ -394,6 +489,20 @@ async def bandwidth_worker(count: int = None):
|
|||
speed=historical[-1][-1], # Use the most recent speed.
|
||||
)
|
||||
})
|
||||
for name, stats in DRIVE_BANDWIDTH.items():
|
||||
if 'historical' not in stats:
|
||||
continue
|
||||
|
||||
historical = stats['historical']
|
||||
bytes_read_ps, bytes_write_ps, elapsed = _calculate_bytes_per_second(historical)
|
||||
DRIVE_BANDWIDTH.update({
|
||||
name: dict(
|
||||
historical=historical,
|
||||
bytes_read_ps=bytes_read_ps,
|
||||
bytes_write_ps=bytes_write_ps,
|
||||
elapsed=elapsed,
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -401,21 +510,26 @@ class Status:
|
|||
cpu_info: CPUInfo
|
||||
load: SystemLoad
|
||||
drives: List[DriveInfo]
|
||||
bandwidth: BandwidthInfo
|
||||
bandwidth: NICBandwidthInfo
|
||||
disk_bandwidth: DiskBandwidthInfo
|
||||
memory_stats: MemoryStats
|
||||
|
||||
|
||||
async def get_status() -> Status:
|
||||
cpu_info, load, drives, bandwidth = await asyncio.gather(
|
||||
cpu_info, load, drives, memory_stats, (nic_bandwidth, disk_bandwidth) = await asyncio.gather(
|
||||
get_cpu_info(),
|
||||
get_load(),
|
||||
get_drives_info(),
|
||||
get_memory_stats(),
|
||||
get_bandwidth_info(),
|
||||
)
|
||||
return Status(
|
||||
cpu_info=cpu_info,
|
||||
load=load,
|
||||
drives=drives,
|
||||
bandwidth=bandwidth,
|
||||
bandwidth=nic_bandwidth,
|
||||
disk_bandwidth=disk_bandwidth,
|
||||
memory_stats=memory_stats,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -428,6 +542,7 @@ if __name__ == '__main__':
|
|||
|
||||
task = asyncio.gather(
|
||||
get_cpu_info(),
|
||||
get_memory_stats(),
|
||||
get_load(),
|
||||
get_drives_info(),
|
||||
get_bandwidth_info(),
|
||||
|
|
|
|||
|
|
@ -4,12 +4,13 @@ import tempfile
|
|||
import unittest
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from itertools import zip_longest
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.common import insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \
|
||||
escape_file_name, match_paths_to_suffixes, truncate_object_bytes, check_media_directory
|
||||
import wrolpi.vars
|
||||
from wrolpi import common
|
||||
from wrolpi.dates import set_timezone, now
|
||||
from wrolpi.errors import InvalidTimezone
|
||||
from wrolpi.test.common import build_test_directories
|
||||
|
|
@ -63,51 +64,51 @@ def test_insert_parameter():
|
|||
def func(foo, bar):
|
||||
pass
|
||||
|
||||
results = insert_parameter(func, 'bar', 'bar', (1,), {})
|
||||
results = common.insert_parameter(func, 'bar', 'bar', (1,), {})
|
||||
assert results == ((1, 'bar'), {})
|
||||
|
||||
def func(foo, bar, baz):
|
||||
pass
|
||||
|
||||
results = insert_parameter(func, 'bar', 'bar', (1, 2), {})
|
||||
results = common.insert_parameter(func, 'bar', 'bar', (1, 2), {})
|
||||
assert results == ((1, 'bar', 2), {})
|
||||
|
||||
def func(foo, baz, bar=None):
|
||||
pass
|
||||
|
||||
results = insert_parameter(func, 'bar', 'bar', (1, 2), {})
|
||||
results = common.insert_parameter(func, 'bar', 'bar', (1, 2), {})
|
||||
assert results == ((1, 2, 'bar'), {})
|
||||
|
||||
def func(foo, baz, bar=None):
|
||||
pass
|
||||
|
||||
results = insert_parameter(func, 'baz', 'baz', (1, 2), {})
|
||||
results = common.insert_parameter(func, 'baz', 'baz', (1, 2), {})
|
||||
assert results == ((1, 'baz', 2), {})
|
||||
|
||||
def func(foo, baz, qux=None, bar=None):
|
||||
pass
|
||||
|
||||
results = insert_parameter(func, 'bar', 'bar', (1, 2, 3), {})
|
||||
results = common.insert_parameter(func, 'bar', 'bar', (1, 2, 3), {})
|
||||
assert results == ((1, 2, 3, 'bar'), {})
|
||||
|
||||
# bar is not defined as a parameter!
|
||||
def func(foo):
|
||||
pass
|
||||
|
||||
pytest.raises(TypeError, insert_parameter, func, 'bar', 'bar', (1,), {})
|
||||
pytest.raises(TypeError, common.insert_parameter, func, 'bar', 'bar', (1,), {})
|
||||
|
||||
|
||||
class TestCommon(unittest.TestCase):
|
||||
|
||||
def test_date_range(self):
|
||||
# A single step results in the start.
|
||||
result = date_range(date(1970, 1, 1), date(1970, 1, 2), 1)
|
||||
result = common.date_range(date(1970, 1, 1), date(1970, 1, 2), 1)
|
||||
assert result == [
|
||||
date(1970, 1, 1),
|
||||
]
|
||||
|
||||
# Many steps on a single day results in the same day.
|
||||
result = date_range(date(1970, 1, 1), date(1970, 1, 1), 5)
|
||||
result = common.date_range(date(1970, 1, 1), date(1970, 1, 1), 5)
|
||||
assert result == [
|
||||
date(1970, 1, 1),
|
||||
date(1970, 1, 1),
|
||||
|
|
@ -117,7 +118,7 @@ class TestCommon(unittest.TestCase):
|
|||
]
|
||||
|
||||
# Many steps on a single datetime results in a range of times.
|
||||
result = date_range(datetime(1970, 1, 1), datetime(1970, 1, 1, 23, 59, 59), 5)
|
||||
result = common.date_range(datetime(1970, 1, 1), datetime(1970, 1, 1, 23, 59, 59), 5)
|
||||
assert result == [
|
||||
datetime(1970, 1, 1, 0, 0),
|
||||
datetime(1970, 1, 1, 4, 47, 59, 800000),
|
||||
|
|
@ -126,8 +127,8 @@ class TestCommon(unittest.TestCase):
|
|||
datetime(1970, 1, 1, 19, 11, 59, 200000),
|
||||
]
|
||||
|
||||
# date_range is not inclusive, like range().
|
||||
result = date_range(date(1970, 1, 1), date(1970, 1, 5), 4)
|
||||
# common.date_range is not inclusive, like range().
|
||||
result = common.date_range(date(1970, 1, 1), date(1970, 1, 5), 4)
|
||||
assert result == [
|
||||
date(1970, 1, 1),
|
||||
date(1970, 1, 2),
|
||||
|
|
@ -136,7 +137,7 @@ class TestCommon(unittest.TestCase):
|
|||
]
|
||||
|
||||
# Reversed dates are supported.
|
||||
result = date_range(date(1970, 1, 5), date(1970, 1, 1), 4)
|
||||
result = common.date_range(date(1970, 1, 5), date(1970, 1, 1), 4)
|
||||
assert result == [
|
||||
date(1970, 1, 5),
|
||||
date(1970, 1, 4),
|
||||
|
|
@ -145,7 +146,7 @@ class TestCommon(unittest.TestCase):
|
|||
]
|
||||
|
||||
# Large date spans are supported.
|
||||
result = date_range(date(1970, 1, 1), date(2020, 5, 1), 4)
|
||||
result = common.date_range(date(1970, 1, 1), date(2020, 5, 1), 4)
|
||||
assert result == [
|
||||
date(1970, 1, 1),
|
||||
date(1982, 8, 1),
|
||||
|
|
@ -153,7 +154,7 @@ class TestCommon(unittest.TestCase):
|
|||
date(2007, 10, 1),
|
||||
]
|
||||
|
||||
result = date_range(datetime(1970, 1, 1, 0, 0, 0), datetime(1970, 1, 1, 10, 0), 8)
|
||||
result = common.date_range(datetime(1970, 1, 1, 0, 0, 0), datetime(1970, 1, 1, 10, 0), 8)
|
||||
assert result == [
|
||||
datetime(1970, 1, 1, 0, 0),
|
||||
datetime(1970, 1, 1, 1, 15),
|
||||
|
|
@ -166,7 +167,7 @@ class TestCommon(unittest.TestCase):
|
|||
]
|
||||
|
||||
# More steps than days
|
||||
result = date_range(date(1970, 1, 1), date(1970, 1, 7), 10)
|
||||
result = common.date_range(date(1970, 1, 1), date(1970, 1, 7), 10)
|
||||
assert result == [
|
||||
date(1970, 1, 1),
|
||||
date(1970, 1, 1),
|
||||
|
|
@ -215,7 +216,7 @@ class TestCommon(unittest.TestCase):
|
|||
]
|
||||
)
|
||||
def test_api_param_limiter(i, expected):
|
||||
limiter = api_param_limiter(100) # should never return an integer greater than 100.
|
||||
limiter = common.api_param_limiter(100) # should never return an integer greater than 100.
|
||||
assert limiter(i) == expected
|
||||
|
||||
|
||||
|
|
@ -227,22 +228,22 @@ def test_chdir():
|
|||
home = os.environ.get('HOME')
|
||||
assert home
|
||||
|
||||
with chdir():
|
||||
with common.chdir():
|
||||
assert os.getcwd() != original
|
||||
assert str(os.getcwd()).startswith('/tmp')
|
||||
assert os.environ['HOME'] != os.getcwd()
|
||||
# Replace $HOME
|
||||
with chdir(with_home=True):
|
||||
with common.chdir(with_home=True):
|
||||
assert os.getcwd() != original
|
||||
assert str(os.getcwd()).startswith('/tmp')
|
||||
assert os.environ['HOME'] == os.getcwd()
|
||||
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
# Without replacing $HOME
|
||||
with chdir(pathlib.Path(d), with_home=True):
|
||||
with common.chdir(pathlib.Path(d), with_home=True):
|
||||
assert os.getcwd() == d
|
||||
assert os.environ['HOME'] == os.getcwd()
|
||||
with chdir(pathlib.Path(d)):
|
||||
with common.chdir(pathlib.Path(d)):
|
||||
assert os.getcwd() == d
|
||||
assert os.environ['HOME'] != os.getcwd()
|
||||
|
||||
|
|
@ -271,7 +272,7 @@ def test_chdir():
|
|||
])
|
||||
])
|
||||
def test_zig_zag(low, high, expected):
|
||||
zagger = zig_zag(low, high)
|
||||
zagger = common.zig_zag(low, high)
|
||||
for i in expected:
|
||||
result = next(zagger)
|
||||
assert result == i
|
||||
|
|
@ -297,7 +298,7 @@ def test_zig_zag(low, high, expected):
|
|||
]
|
||||
)
|
||||
def test_escape_file_name(name, expected):
|
||||
assert escape_file_name(name) == expected
|
||||
assert common.escape_file_name(name) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -319,37 +320,121 @@ def test_escape_file_name(name, expected):
|
|||
def test_match_paths_to_suffixes(paths, suffix_groups, expected):
|
||||
paths = [pathlib.Path(i) for i in paths]
|
||||
expected = tuple(pathlib.Path(i) if i else None for i in expected)
|
||||
assert (i := match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}'
|
||||
assert (i := common.match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}'
|
||||
|
||||
|
||||
def test_truncate_object_bytes():
|
||||
"""
|
||||
Objects can be truncated (lists will be shortened) so they will fit in tsvector columns.
|
||||
"""
|
||||
assert truncate_object_bytes(['foo'] * 10, 100) == ['foo'] * 5
|
||||
assert truncate_object_bytes(['foo'] * 1_000, 100) == ['foo'] * 5
|
||||
assert truncate_object_bytes(['foo'] * 1_000_000, 100) == ['foo'] * 5
|
||||
assert truncate_object_bytes(['foo'] * 1_000_000, 200) == ['foo'] * 14
|
||||
assert truncate_object_bytes([], 200) == []
|
||||
assert common.truncate_object_bytes(['foo'] * 10, 100) == ['foo'] * 5
|
||||
assert common.truncate_object_bytes(['foo'] * 1_000, 100) == ['foo'] * 5
|
||||
assert common.truncate_object_bytes(['foo'] * 1_000_000, 100) == ['foo'] * 5
|
||||
assert common.truncate_object_bytes(['foo'] * 1_000_000, 200) == ['foo'] * 14
|
||||
assert common.truncate_object_bytes([], 200) == []
|
||||
|
||||
assert truncate_object_bytes(None, 100) is None
|
||||
assert truncate_object_bytes('', 100) == ''
|
||||
assert common.truncate_object_bytes(None, 100) is None
|
||||
assert common.truncate_object_bytes('', 100) == ''
|
||||
|
||||
assert truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof'
|
||||
assert truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo'
|
||||
assert truncate_object_bytes('foo' * 100, 55) == 'foofo'
|
||||
assert truncate_object_bytes('foo' * 100, 51) == 'f'
|
||||
assert truncate_object_bytes('foo' * 100, 50) == ''
|
||||
assert truncate_object_bytes('foo' * 100, 0) == ''
|
||||
assert common.truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof'
|
||||
assert common.truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo'
|
||||
assert common.truncate_object_bytes('foo' * 100, 55) == 'foofo'
|
||||
assert common.truncate_object_bytes('foo' * 100, 51) == 'f'
|
||||
assert common.truncate_object_bytes('foo' * 100, 50) == ''
|
||||
assert common.truncate_object_bytes('foo' * 100, 0) == ''
|
||||
|
||||
|
||||
def test_check_media_directory(test_directory):
|
||||
"""The directory provided by the test_directory fixture is a valid media directory."""
|
||||
assert check_media_directory() is True
|
||||
assert common.check_media_directory() is True
|
||||
|
||||
|
||||
def test_bad_check_media_directory():
|
||||
"""/dev/full is not a valid media directory, warnings are issued and the check fails."""
|
||||
with mock.patch('wrolpi.common.get_media_directory') as mock_get_media_directory:
|
||||
mock_get_media_directory.return_value = pathlib.Path('/dev/full')
|
||||
assert check_media_directory() is False
|
||||
assert common.check_media_directory() is False
|
||||
|
||||
|
||||
def test_chunks_by_name(test_directory, make_files_structure):
|
||||
"""`chunks_by_name` breaks a list of paths on the name change close to the size."""
|
||||
with pytest.raises(ValueError):
|
||||
assert list(common.chunks_by_name([], 0)) == [[]]
|
||||
|
||||
assert list(common.chunks_by_name([], 5)) == [[]]
|
||||
assert list(common.chunks_by_name([1, 2, 3], 5)) == [[1, 2, 3]]
|
||||
|
||||
files = make_files_structure([
|
||||
'foo.mp4', 'foo.txt', 'foo.png', 'foo.info.json',
|
||||
'bar.mp4', 'bar.readability.txt', 'bar.jpeg', 'bar.info.json',
|
||||
'baz.mp4', 'baz.txt', 'baz.jpg', 'baz.info.json',
|
||||
'qux.mp4', 'qux.txt', 'qux.tif', 'qux.info.json',
|
||||
])
|
||||
|
||||
def assert_chunks(size, files_, expected):
|
||||
for chunk, expected_chunk in zip_longest(list(common.chunks_by_name(files_, size)), expected):
|
||||
assert chunk == [test_directory / i for i in expected_chunk]
|
||||
|
||||
assert_chunks(8, files, [
|
||||
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt',
|
||||
'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt',
|
||||
'foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'],
|
||||
['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
|
||||
])
|
||||
|
||||
assert_chunks(6, files, [
|
||||
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt',
|
||||
'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
|
||||
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt',
|
||||
'qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
|
||||
])
|
||||
|
||||
assert_chunks(5, files, [
|
||||
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt',
|
||||
'baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
|
||||
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt',
|
||||
'qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
|
||||
])
|
||||
|
||||
assert_chunks(3, files, [
|
||||
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt'],
|
||||
['baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
|
||||
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'],
|
||||
['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
|
||||
])
|
||||
|
||||
assert_chunks(1, files, [
|
||||
['bar.info.json', 'bar.jpeg', 'bar.mp4', 'bar.readability.txt'],
|
||||
['baz.info.json', 'baz.jpg', 'baz.mp4', 'baz.txt'],
|
||||
['foo.info.json', 'foo.mp4', 'foo.png', 'foo.txt'],
|
||||
['qux.info.json', 'qux.mp4', 'qux.tif', 'qux.txt'],
|
||||
])
|
||||
|
||||
files = make_files_structure(['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4'])
|
||||
assert_chunks(1, files, [['1.mp4', '1.txt'], ['2.mp4', '2.png', '2.txt'], ['3.mp4']])
|
||||
assert_chunks(3, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt'], ['3.mp4']])
|
||||
assert_chunks(6, files, [['1.mp4', '1.txt', '2.mp4', '2.png', '2.txt', '3.mp4']])
|
||||
# List is not sorted because it is less than the size.
|
||||
assert_chunks(20, files, [['1.mp4', '1.txt', '2.mp4', '2.txt', '2.png', '3.mp4']])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'value,expected', [
|
||||
('true', True),
|
||||
('t', True),
|
||||
('True', True),
|
||||
('1', True),
|
||||
('yes', True),
|
||||
('no', False),
|
||||
('0', False),
|
||||
('False', False),
|
||||
('false', False),
|
||||
('f', False),
|
||||
('other', False),
|
||||
('trust', False),
|
||||
('', False),
|
||||
(None, False),
|
||||
]
|
||||
)
|
||||
def test_truthy_arg(value, expected):
|
||||
assert wrolpi.vars.truthy_arg(value) is expected, f'{value} != {expected}'
|
||||
|
|
|
|||
|
|
@ -314,6 +314,7 @@ def test_get_status(test_client, test_session):
|
|||
assert 'hotspot_status' in response.json and isinstance(response.json['hotspot_status'], str)
|
||||
assert 'throttle_status' in response.json and isinstance(response.json['throttle_status'], str)
|
||||
assert 'version' in response.json and isinstance(response.json['version'], str)
|
||||
assert 'memory_stats' in response.json and isinstance(response.json['memory_stats'], dict)
|
||||
|
||||
|
||||
def test_post_download(test_session, test_client, test_download_manager_config):
|
||||
|
|
|
|||
|
|
@ -59,22 +59,26 @@ async def test_get_drives_info():
|
|||
@pytest.mark.asyncio
|
||||
async def test_get_bandwidth_info():
|
||||
"""Bandwidth requires psutil"""
|
||||
bandwidths = await status.get_bandwidth_info()
|
||||
assert bandwidths == []
|
||||
nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info()
|
||||
assert nic_bandwidths == []
|
||||
assert disk_bandwidths == []
|
||||
|
||||
await status.bandwidth_worker(2)
|
||||
bandwidths = await status.get_bandwidth_info()
|
||||
assert isinstance(bandwidths, list)
|
||||
assert len(bandwidths) > 0
|
||||
assert isinstance(bandwidths[0], status.BandwidthInfo)
|
||||
nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info()
|
||||
assert isinstance(nic_bandwidths, list)
|
||||
assert len(nic_bandwidths) > 0
|
||||
assert isinstance(nic_bandwidths[0], status.NICBandwidthInfo)
|
||||
assert isinstance(disk_bandwidths, list)
|
||||
assert len(disk_bandwidths) > 0
|
||||
assert isinstance(disk_bandwidths[0], status.DiskBandwidthInfo)
|
||||
|
||||
with mock.patch('wrolpi.status.psutil.net_io_counters') as mock_net_io_counters:
|
||||
# NO FALLBACK!
|
||||
mock_net_io_counters.side_effect = Exception('testing no psutil')
|
||||
await status.bandwidth_worker(1)
|
||||
bandwidths = await status.get_bandwidth_info()
|
||||
assert isinstance(bandwidths[0].bytes_recv, int)
|
||||
assert isinstance(bandwidths[0].bytes_sent, int)
|
||||
assert isinstance(bandwidths[0].elapsed, int)
|
||||
assert isinstance(bandwidths[0].speed, int)
|
||||
assert isinstance(bandwidths[0].name, str)
|
||||
nic_bandwidths, disk_bandwidths = await status.get_bandwidth_info()
|
||||
assert isinstance(nic_bandwidths[0].bytes_recv, int)
|
||||
assert isinstance(nic_bandwidths[0].bytes_sent, int)
|
||||
assert isinstance(nic_bandwidths[0].elapsed, int)
|
||||
assert isinstance(nic_bandwidths[0].speed, int)
|
||||
assert isinstance(nic_bandwidths[0].name, str)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
|
@ -15,6 +16,33 @@ DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else F
|
|||
# tests are running
|
||||
PYTEST = 'pytest' in sys.modules
|
||||
|
||||
|
||||
def truthy_arg(value: str) -> bool:
|
||||
"""Attempts to convert `value` from console arguments to a boolean value.
|
||||
|
||||
>>> truthy_arg('true')
|
||||
True
|
||||
>>> truthy_arg('True')
|
||||
True
|
||||
>>> truthy_arg('t')
|
||||
True
|
||||
>>> truthy_arg('1')
|
||||
True
|
||||
>>> truthy_arg('yes')
|
||||
True
|
||||
>>> truthy_arg('0')
|
||||
False
|
||||
>>> truthy_arg('false')
|
||||
False
|
||||
>>> truthy_arg('f')
|
||||
False
|
||||
>>> truthy_arg('no')
|
||||
False
|
||||
"""
|
||||
v = str(value).lower()
|
||||
return v == 'true' or v == 't' or v == '1' or v == 'yes'
|
||||
|
||||
|
||||
# Get the media directory from the environment.
|
||||
DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
|
||||
MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
|
||||
|
|
@ -37,3 +65,12 @@ DB_PORT = int(os.environ.get('DB_PORT', 5432))
|
|||
DB_NAME = os.environ.get('DB_NAME', 'wrolpi')
|
||||
DB_USER = os.environ.get('DB_USER', 'wrolpi')
|
||||
DB_PASSWORD = os.environ.get('DB_PASSWORD', 'wrolpi')
|
||||
|
||||
API_HOST = os.environ.get('API_HOST', '127.0.0.1')
|
||||
API_PORT = os.environ.get('API_PORT', '8081')
|
||||
API_WORKERS = os.environ.get('API_WORKERS', multiprocessing.cpu_count())
|
||||
API_AUTO_RELOAD = truthy_arg(os.environ.get('API_AUTO_RELOAD', DOCKERIZED))
|
||||
API_ACCESS_LOG = truthy_arg(os.environ.get('API_ACCESS_LOG', DOCKERIZED))
|
||||
API_DEBUG = truthy_arg(os.environ.get('API_DEBUG', False))
|
||||
|
||||
FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
__version__ = '0.7.0-beta'
|
||||
__version__ = '0.7.1-beta'
|
||||
|
||||
import subprocess
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue