mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Using iostat to detect busy disk issues, reporting them in the UI. Detecting under-voltage/over-current issues using dmesg and reporting them in the UI.
This commit is contained in:
parent
220cb8010f
commit
d8b3db9d63
14 changed files with 339 additions and 282 deletions
|
|
@ -20,15 +20,7 @@ import {
|
|||
} from 'semantic-ui-react';
|
||||
import {Link, NavLink, useNavigate, useSearchParams} from "react-router-dom";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {
|
||||
useCPUTemperature,
|
||||
useHotspot,
|
||||
useLoad,
|
||||
useSearchDirectories,
|
||||
useSearchOrder,
|
||||
useThrottle,
|
||||
useWROLMode
|
||||
} from "../hooks/customHooks";
|
||||
import {useHotspot, useSearchDirectories, useSearchOrder, useThrottle, useWROLMode} from "../hooks/customHooks";
|
||||
import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts";
|
||||
import {
|
||||
Accordion,
|
||||
|
|
@ -1124,38 +1116,6 @@ export function HotspotStatusIcon() {
|
|||
</>
|
||||
}
|
||||
|
||||
export function CPUTemperatureIcon({size = 'large', fallback = null}) {
|
||||
// Returns an Icon only if temperature is too high.
|
||||
const {temperature, highTemperature, criticalTemperature} = useCPUTemperature();
|
||||
|
||||
if (temperature === null || temperature < highTemperature) {
|
||||
// Temperature is not a problem.
|
||||
return fallback;
|
||||
}
|
||||
|
||||
let icon = temperature >= criticalTemperature ?
|
||||
<Icon data-testid='cpuTemperatureIcon' name='thermometer' size={size} color='red'/>
|
||||
: <Icon data-testid='cpuTemperatureIcon' name='thermometer half' size={size} color='yellow'/>;
|
||||
icon = <Link to='/admin/status'>{icon}</Link>;
|
||||
|
||||
return <Popup content={`CPU: ${temperature}°C`} trigger={icon}/>
|
||||
}
|
||||
|
||||
export function SystemLoadIcon({size = 'large', fallback = null}) {
|
||||
const {minute_1, mediumLoad, highLoad} = useLoad();
|
||||
|
||||
if (!mediumLoad && !highLoad) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
// Only return the icon if the load is not low.
|
||||
const color = highLoad ? 'red' : 'yellow';
|
||||
let icon = <Link to='/admin/status'>
|
||||
<Icon name='tachometer alternate' size={size} color={color}/>
|
||||
</Link>;
|
||||
return <Popup content={`Load: ${minute_1}`} trigger={icon}/>
|
||||
}
|
||||
|
||||
export function useTitle(title) {
|
||||
const documentDefined = typeof document !== 'undefined';
|
||||
const originalTitle = React.useRef(documentDefined ? document.title : null);
|
||||
|
|
@ -1950,3 +1910,11 @@ export function getDistinctColor(hexColors) {
|
|||
// If we've tried maxAttempts times, return the last generated color regardless
|
||||
return hslToHex(newColor.h, newColor.s, newColor.l);
|
||||
}
|
||||
|
||||
export const isReactChildNull = (children) => {
|
||||
// Returns `true` if a React element returns `null` as it's child.
|
||||
const nonNullChildren = React.Children.map(children,
|
||||
(child) => React.isValidElement(child) && child.type(child.props)
|
||||
);
|
||||
return nonNullChildren.length === 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ const ViewerMessage = () => {
|
|||
|
||||
function SlowImportMessage() {
|
||||
const {status} = useContext(StatusContext);
|
||||
if (status && status['cpu_info'] && status['cpu_info']['temperature'] >= 80) {
|
||||
if (status && status['cpu_stats'] && status['cpu_stats']['temperature'] >= 80) {
|
||||
return <Message warning icon>
|
||||
<SIcon name='exclamation'/>
|
||||
<Message.Content>
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ import React from "react";
|
|||
import {Link, NavLink} from "react-router-dom";
|
||||
import {Dropdown, Icon as SIcon, Menu} from "semantic-ui-react";
|
||||
import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts";
|
||||
import {CPUTemperatureIcon, DarkModeToggle, HotspotStatusIcon, SystemLoadIcon, useLocalStorage} from "./Common";
|
||||
import {DarkModeToggle, HotspotStatusIcon, isReactChildNull, useLocalStorage} from "./Common";
|
||||
import {ShareButton} from "./Share";
|
||||
import {useWROLMode} from "../hooks/customHooks";
|
||||
import {useCPUTemperature, useIOStats, useLoad, useMemoryStats, usePowerStats, useWROLMode} from "../hooks/customHooks";
|
||||
import {SearchIconButton} from "./Search";
|
||||
import {Icon} from "./Theme";
|
||||
import {Icon, Popup} from "./Theme";
|
||||
import {HELP_VIEWER_URI, NAME} from "./Vars";
|
||||
import _ from "lodash";
|
||||
|
||||
|
|
@ -59,15 +59,11 @@ function MenuLink({link}) {
|
|||
}
|
||||
}
|
||||
|
||||
function NavIconWrapper(props) {
|
||||
if (_.isEmpty(props.children)) {
|
||||
return <></>
|
||||
} else {
|
||||
if (props.name === 'warning') {
|
||||
// console.log(props.name, props.children);
|
||||
}
|
||||
return <div style={{marginTop: '0.8em', marginLeft: '1.5em'}}>{props.children}</div>
|
||||
}
|
||||
function NavIconWrapper({children}) {
|
||||
// Do not add margins around empty react element to avoid filling up navbar.
|
||||
const emptyChild = !children || isReactChildNull(children);
|
||||
const style = emptyChild ? {} : {marginTop: '0.8em', marginLeft: '1.5em'};
|
||||
return <div style={style}>{children}</div>
|
||||
}
|
||||
|
||||
function useNavColorSetting() {
|
||||
|
|
@ -97,8 +93,74 @@ export function NavBar() {
|
|||
{topNavText}
|
||||
</NavLink>;
|
||||
|
||||
// Display the temperature icon first because it can cause the system to throttle.
|
||||
const warningIcon = <CPUTemperatureIcon fallback={<SystemLoadIcon/>}/>;
|
||||
// Red/Yellow colors will blend in to some navbar colors, swap the colors for high contrast.
|
||||
const conflictingColors = ['red', 'orange', 'yellow', 'olive', 'pink'];
|
||||
const lowWarningColor = conflictingColors.includes(navColor) ? 'white' : 'yellow';
|
||||
const highWarningColor = conflictingColors.includes(navColor) ? 'black' : 'red';
|
||||
|
||||
// Generic system load, the least important warning.
|
||||
const {minute_1, mediumLoad, highLoad} = useLoad();
|
||||
let systemLoadIcon;
|
||||
if (mediumLoad || highLoad) {
|
||||
// System load is high, display a warning icon.
|
||||
const color = highLoad ? highWarningColor : lowWarningColor;
|
||||
const icon = <Link to='/admin/status'>
|
||||
<Icon name='tachometer alternate' size='large' color={color}/>
|
||||
</Link>;
|
||||
systemLoadIcon = <Popup content={`Load: ${minute_1}`} trigger={icon}/>
|
||||
}
|
||||
|
||||
// RAM consumption.
|
||||
const {percent: memoryPercent} = useMemoryStats();
|
||||
let memoryIcon;
|
||||
if (memoryPercent > 80) {
|
||||
const color = memoryPercent > 90 ? highWarningColor : lowWarningColor;
|
||||
const icon = <Link to='/admin/status' color={color}>
|
||||
<Icon name='microchip' size='large'/>
|
||||
</Link>;
|
||||
memoryIcon = <Popup content={`System Memory: ${memoryPercent}%`} trigger={icon}/>
|
||||
}
|
||||
|
||||
// Any disk is busy and processes are waiting.
|
||||
const {percentIOWait} = useIOStats();
|
||||
let diskWaitIcon;
|
||||
if (percentIOWait >= 50) {
|
||||
// Processes are waiting on disk, display a warning icon.
|
||||
const color = percentIOWait > 75 ? highWarningColor : lowWarningColor;
|
||||
const icon = <Link to='/admin/status' color={color}>
|
||||
<Icon name='disk' size='large'/>
|
||||
</Link>;
|
||||
diskWaitIcon = <Popup content={`Processes waiting on disk: ${percentIOWait}`} trigger={icon}/>
|
||||
}
|
||||
|
||||
// CPU temperature.
|
||||
const {temperature, highTemperature, criticalTemperature} = useCPUTemperature();
|
||||
let temperatureIcon;
|
||||
if (temperature && temperature >= highTemperature) {
|
||||
// CPU temperature is high, display a warning icon.
|
||||
const color = temperature >= criticalTemperature ? highWarningColor : lowWarningColor;
|
||||
const name = temperature >= criticalTemperature ? 'thermometer' : 'thermometer half';
|
||||
const icon = <Icon data-testid='cpuTemperatureIcon' name={name} size='large' color={color}/>
|
||||
const link = <Link to='/admin/status'>{icon}</Link>;
|
||||
temperatureIcon = <Popup content={`CPU: ${temperature}°C`} trigger={link}/>;
|
||||
}
|
||||
|
||||
// Power issues, this is always displayed if detected.
|
||||
const {underVoltage, overCurrent} = usePowerStats();
|
||||
let powerIcon;
|
||||
if (underVoltage || overCurrent) {
|
||||
const name = underVoltage ? 'power cord' : 'lightning';
|
||||
const icon = <Icon name={name} size='large' color={highWarningColor}/>;
|
||||
const message = underVoltage
|
||||
? 'Under-voltage detected! Your power supply is insufficient!'
|
||||
: 'Over-current detected! Your peripherals are using too much power!';
|
||||
powerIcon = <Popup content={message} trigger={icon}/>;
|
||||
}
|
||||
|
||||
// Display the temperature icon first because it can cause the system to throttle. The rest are in order of effects
|
||||
// that will slow down the system and the user should address. Generic load is last because it is probably not an
|
||||
// issue.
|
||||
const warningIcon = temperatureIcon || diskWaitIcon || memoryIcon || systemLoadIcon;
|
||||
|
||||
let processingLink;
|
||||
if (status && status.flags) {
|
||||
|
|
@ -115,11 +177,12 @@ export function NavBar() {
|
|||
</Link>;
|
||||
|
||||
const icons = <React.Fragment>
|
||||
<NavIconWrapper name='processing'>{processingIcon}</NavIconWrapper>
|
||||
<NavIconWrapper name='warning'>{warningIcon}</NavIconWrapper>
|
||||
<NavIconWrapper name='share'><ShareButton/></NavIconWrapper>
|
||||
<NavIconWrapper name='hotspot'><HotspotStatusIcon/></NavIconWrapper>
|
||||
<NavIconWrapper name='dark mode'><DarkModeToggle/></NavIconWrapper>
|
||||
<NavIconWrapper>{processingIcon}</NavIconWrapper>
|
||||
<NavIconWrapper>{powerIcon}</NavIconWrapper>
|
||||
<NavIconWrapper>{warningIcon}</NavIconWrapper>
|
||||
<NavIconWrapper><ShareButton/></NavIconWrapper>
|
||||
<NavIconWrapper><HotspotStatusIcon/></NavIconWrapper>
|
||||
<NavIconWrapper><DarkModeToggle/></NavIconWrapper>
|
||||
</React.Fragment>;
|
||||
|
||||
return <>
|
||||
|
|
|
|||
|
|
@ -877,39 +877,29 @@ export const StatusProvider = (props) => {
|
|||
|
||||
export const useStatusFlag = (flag) => {
|
||||
const {status} = useContext(StatusContext);
|
||||
return status && status['flags'] && status['flags'][flag];
|
||||
return status?.flags?.[flag] || false;
|
||||
}
|
||||
|
||||
export const useCPUTemperature = () => {
|
||||
const {status} = React.useContext(StatusContext);
|
||||
let temperature = 0;
|
||||
let highTemperature = 75;
|
||||
let criticalTemperature = 85;
|
||||
|
||||
if (status && status['cpu_info']) {
|
||||
temperature = status['cpu_info']['temperature'];
|
||||
highTemperature = status['cpu_info']['high_temperature'];
|
||||
criticalTemperature = status['cpu_info']['critical_temperature'];
|
||||
}
|
||||
|
||||
const temperature = status?.cpu_stats?.temperature || 0;
|
||||
const highTemperature = status?.cpu_stats?.high_temperature || 75;
|
||||
const criticalTemperature = status?.cpu_stats?.critical_temperature || 85;
|
||||
return {temperature, highTemperature, criticalTemperature}
|
||||
}
|
||||
|
||||
export const useLoad = () => {
|
||||
const {status} = React.useContext(StatusContext);
|
||||
let minute_1;
|
||||
let minute_5;
|
||||
let minute_15;
|
||||
let minute_1 = status?.load_stats?.minute_1;
|
||||
let minute_5 = status?.load_stats?.minute_5;
|
||||
let minute_15 = status?.load_stats?.minute_15;
|
||||
let mediumLoad = false;
|
||||
let highLoad = false;
|
||||
let cores;
|
||||
|
||||
if (status && status['load']) {
|
||||
minute_1 = status['load']['minute_1'];
|
||||
minute_5 = status['load']['minute_5'];
|
||||
minute_15 = status['load']['minute_15'];
|
||||
|
||||
cores = status['cpu_info']['cores'];
|
||||
if (status && status.cpu_stats) {
|
||||
// Medium load when 1/2 cores are busy, High load when 3/4+ cores are busy.
|
||||
cores = status.cpu_stats.cores;
|
||||
const quarter = cores / 4;
|
||||
if (cores && minute_1 >= (quarter * 3)) {
|
||||
highLoad = true;
|
||||
|
|
@ -921,6 +911,37 @@ export const useLoad = () => {
|
|||
return {minute_1, minute_5, minute_15, mediumLoad, highLoad, cores};
|
||||
}
|
||||
|
||||
export const useIOStats = () => {
|
||||
const {status} = React.useContext(StatusContext);
|
||||
const percentIdle = status?.iostat_stats?.percent_idle;
|
||||
const percentIOWait = status?.iostat_stats?.percent_iowait;
|
||||
const percentNice = status?.iostat_stats?.percent_nice;
|
||||
const percentSteal = status?.iostat_stats?.percent_steal;
|
||||
const percentSystem = status?.iostat_stats?.percent_system;
|
||||
const percentUser = status?.iostat_stats?.percent_user;
|
||||
|
||||
return {percentIdle, percentIOWait, percentNice, percentSteal, percentSystem, percentUser};
|
||||
}
|
||||
|
||||
export const useMemoryStats = () => {
|
||||
const {status} = React.useContext(StatusContext);
|
||||
const total = status?.memory_stats?.total;
|
||||
const used = status?.memory_stats?.used;
|
||||
const free = status?.memory_stats?.free;
|
||||
const cached = status?.memory_stats?.cached;
|
||||
let percent;
|
||||
if (total && used) {
|
||||
percent = Math.round((used / total) * 100);
|
||||
}
|
||||
return {total, used, free, cached, percent}
|
||||
}
|
||||
|
||||
export const usePowerStats = () => {
|
||||
const {status} = React.useContext(StatusContext);
|
||||
const underVoltage = status?.power_stats?.under_voltage;
|
||||
const overCurrent = status?.power_stats?.over_current;
|
||||
return {underVoltage, overCurrent};
|
||||
}
|
||||
|
||||
export const useVideoStatistics = () => {
|
||||
const [statistics, setStatistics] = useState(null);
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ rsync
|
|||
smartmontools
|
||||
software-properties-common
|
||||
sudo
|
||||
sysstat
|
||||
tar
|
||||
tcpdump
|
||||
tmux
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ WORKDIR /opt/wrolpi
|
|||
# Update dependencies for the services install.
|
||||
RUN apt update
|
||||
# ffmpeg for Videos
|
||||
RUN apt-get install -y ffmpeg catdoc
|
||||
RUN apt-get install -y ffmpeg catdoc sysstat
|
||||
RUN ffmpeg -version
|
||||
# yt-dlp needs cache directory
|
||||
RUN mkdir /.cache && chown -R 1000:1000 /.cache
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ renderd
|
|||
rsync
|
||||
smartmontools
|
||||
software-properties-common
|
||||
sysstat
|
||||
tar
|
||||
tcpdump
|
||||
tmux
|
||||
|
|
|
|||
148
wrolpi/common.py
148
wrolpi/common.py
|
|
@ -21,7 +21,6 @@ from dataclasses import dataclass, asdict, field, fields
|
|||
from datetime import datetime, date
|
||||
from decimal import Decimal
|
||||
from functools import wraps
|
||||
from http import HTTPStatus
|
||||
from itertools import islice, filterfalse, tee
|
||||
from multiprocessing.managers import DictProxy
|
||||
from pathlib import Path
|
||||
|
|
@ -40,8 +39,8 @@ from sqlalchemy.ext.compiler import compiles
|
|||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
|
||||
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError, InvalidConfig, \
|
||||
from wrolpi.dates import now, from_timestamp
|
||||
from wrolpi.errors import WROLModeEnabled, NativeOnly, LogLevelError, InvalidConfig, \
|
||||
ValidationError
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS
|
||||
|
||||
|
|
@ -240,6 +239,7 @@ __all__ = [
|
|||
'apply_modelers',
|
||||
'apply_refresh_cleanup',
|
||||
'background_task',
|
||||
'can_connect_to_server',
|
||||
'cancel_background_tasks',
|
||||
'cancel_refresh_tasks',
|
||||
'cancelable_wrapper',
|
||||
|
|
@ -249,10 +249,10 @@ __all__ = [
|
|||
'chunks',
|
||||
'chunks_by_stem',
|
||||
'compile_tsvector',
|
||||
'create_empty_config_files',
|
||||
'cum_timer',
|
||||
'date_range',
|
||||
'disable_wrol_mode',
|
||||
'download_file',
|
||||
'enable_wrol_mode',
|
||||
'escape_file_name',
|
||||
'extract_domain',
|
||||
|
|
@ -1414,48 +1414,6 @@ async def aiohttp_head(url: str, timeout: int = None, headers: dict = None) -> C
|
|||
yield response
|
||||
|
||||
|
||||
async def speed_test(url: str, timeout: int | None = 10) -> int:
|
||||
"""Request the last megabyte of the provided url, return the content length divided by the time elapsed (speed)."""
|
||||
async with aiohttp_head(url) as response:
|
||||
content_length = response.headers['Content-Length']
|
||||
start_bytes = int(content_length) - 10485760
|
||||
range_ = f'bytes={start_bytes}-{content_length}'
|
||||
start_time = datetime.now()
|
||||
async with aiohttp_get(url, headers={'Range': range_}, timeout=timeout) as response:
|
||||
content = await response.content.read()
|
||||
elapsed = int((datetime.now() - start_time).total_seconds())
|
||||
return len(content) // elapsed
|
||||
|
||||
|
||||
async def get_fastest_mirror(urls: List[str]) -> str:
|
||||
"""Perform a speed test on each URL, return the fastest."""
|
||||
fastest_url = urls[0]
|
||||
fastest_speed = 0
|
||||
for url in urls:
|
||||
try:
|
||||
# Timeout after 5 minutes. Mirror is less than 27 kbps.
|
||||
speed = await speed_test(url, timeout=60 * 5)
|
||||
download_logger.debug(f'Speed test of {url} was {speed} bps')
|
||||
except Exception as e:
|
||||
download_logger.error(f'Speedtest of {url} failed', exc_info=e)
|
||||
continue
|
||||
|
||||
if speed > fastest_speed:
|
||||
fastest_url = url
|
||||
|
||||
return fastest_url
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadFileInfoLink:
|
||||
"""Represents an HTTP "Link" Header."""
|
||||
url: str
|
||||
rel: str
|
||||
type: str
|
||||
priority: int
|
||||
geo: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadFileInfo:
|
||||
"""Information about a file that can be downloaded."""
|
||||
|
|
@ -1465,7 +1423,6 @@ class DownloadFileInfo:
|
|||
accept_ranges: str = None
|
||||
status: int = None
|
||||
location: str = None
|
||||
links: List[DownloadFileInfoLink] = None
|
||||
|
||||
|
||||
FILENAME_MATCHER = re.compile(r'.*filename="(.*)"')
|
||||
|
|
@ -1475,28 +1432,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
|
|||
"""Gets information (name, size, etc.) about a downloadable file at the provided URL."""
|
||||
async with aiohttp_head(url, timeout) as response:
|
||||
download_logger.debug(f'{response.headers=}')
|
||||
try:
|
||||
links = response.headers.getall('Link')
|
||||
except KeyError:
|
||||
links = None
|
||||
|
||||
new_links = list()
|
||||
if links:
|
||||
# Convert "Link" header strings to DownloadFileInfoLink.
|
||||
for idx, link in enumerate(links):
|
||||
url, *props = link.split(';')
|
||||
url = url[1:-1]
|
||||
properties = dict()
|
||||
for prop in props:
|
||||
name, value = prop.strip().split('=')
|
||||
properties[name] = value
|
||||
new_links.append(DownloadFileInfoLink(
|
||||
url,
|
||||
properties.get('rel').strip() if 'rel' in properties else None,
|
||||
properties.get('type').strip() if 'type' in properties else None,
|
||||
int(properties.get('pri').strip()) if 'pri' in properties else None,
|
||||
properties.get('geo').strip() if 'geo' in properties else None,
|
||||
))
|
||||
|
||||
info = DownloadFileInfo(
|
||||
type=response.headers.get('Content-Type'),
|
||||
|
|
@ -1504,7 +1439,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
|
|||
accept_ranges=response.headers.get('Accept-Ranges'),
|
||||
status=response.status,
|
||||
location=response.headers.get('Location'),
|
||||
links=new_links,
|
||||
)
|
||||
|
||||
disposition = response.headers.get('Content-Disposition')
|
||||
|
|
@ -1526,80 +1460,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
|
|||
download_logger = logger.getChild('download')
|
||||
|
||||
|
||||
async def download_file(url: str, output_path: pathlib.Path = None, info: DownloadFileInfo = None,
|
||||
timeout: int = 7 * 24 * 60 * 60):
|
||||
"""Uses aiohttp to download an HTTP file. Performs a speed test when mirrors are found, downloads from the fastest
|
||||
mirror.
|
||||
|
||||
Attempts to resume the file if `output_path` already exists.
|
||||
|
||||
@warning: Timeout default is a week because of large downloads.
|
||||
"""
|
||||
info = info or await get_download_info(url, timeout)
|
||||
|
||||
if output_path.is_file() and info.size == output_path.stat().st_size:
|
||||
download_logger.warning(f'Already downloaded {repr(str(url))} to {repr(str(output_path))}')
|
||||
return
|
||||
|
||||
if info.links and (mirror_urls := [i.url for i in info.links if i.rel == 'duplicate']):
|
||||
# Mirrors are available, find the fastest.
|
||||
download_logger.info(f'Performing download speed test on mirrors: {mirror_urls}')
|
||||
url = await get_fastest_mirror(mirror_urls)
|
||||
info = await get_download_info(url, timeout)
|
||||
|
||||
download_logger.debug(f'Final DownloadInfo fetched {info}')
|
||||
total_size = info.size
|
||||
|
||||
download_logger.info(f'Starting download of {url} with {total_size} total bytes')
|
||||
if info.accept_ranges == 'bytes' or not output_path.is_file():
|
||||
with output_path.open('ab') as fh:
|
||||
headers = DEFAULT_HTTP_HEADERS.copy()
|
||||
# Check the position of append, if it is 0 then we do not need to resume.
|
||||
position = fh.tell()
|
||||
if position:
|
||||
headers['Range'] = f'bytes={position}-'
|
||||
|
||||
async with aiohttp_session(timeout=timeout) as session:
|
||||
async with session.get(url, headers=headers) as response:
|
||||
response: ClientResponse
|
||||
if response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
|
||||
download_logger.warning(f'Server responded with 416, file is probably already downloaded')
|
||||
return
|
||||
|
||||
if position and response.status != HTTPStatus.PARTIAL_CONTENT:
|
||||
raise UnrecoverableDownloadError(
|
||||
f'Tried to resume {repr(str(url))} but got status {response.status}')
|
||||
|
||||
# May or may not be using Range. Append each chunk to the output file.
|
||||
last_report = datetime.now()
|
||||
bytes_received = 0
|
||||
async for data in response.content.iter_any():
|
||||
fh.write(data)
|
||||
|
||||
bytes_received += len(data)
|
||||
if (elapsed := (datetime.now() - last_report).total_seconds()) > 10:
|
||||
# Report download speed every 10 seconds.
|
||||
bytes_per_second = int(bytes_received // elapsed)
|
||||
download_logger.debug(f'{bytes_received=} {elapsed=} {bytes_per_second=}')
|
||||
size = fh.tell()
|
||||
bytes_remaining = total_size - size
|
||||
seconds_remaining = bytes_remaining // bytes_per_second
|
||||
percent = int((size / total_size) * 100)
|
||||
download_logger.info(
|
||||
f'Downloading {url} at'
|
||||
f' rate={human_bandwidth(bytes_per_second)}'
|
||||
f' estimate={seconds_to_timestamp(seconds_remaining)}')
|
||||
download_logger.debug(f'Downloading {url} {total_size=} {size=} {percent=}')
|
||||
last_report = datetime.now()
|
||||
bytes_received = 0
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
elif output_path.is_file():
|
||||
# TODO support downloading files that cannot be resumed.
|
||||
raise UnrecoverableDownloadError(f'Cannot resume download {url}')
|
||||
|
||||
|
||||
def human_bandwidth(bps: int) -> str:
|
||||
"""Convert bits per second to a more readable format.
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import logging
|
|||
import multiprocessing
|
||||
import pathlib
|
||||
import shutil
|
||||
import socketserver
|
||||
import tempfile
|
||||
import threading
|
||||
import zipfile
|
||||
|
|
@ -760,26 +759,51 @@ def make_multipart_form():
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def simple_web_server():
|
||||
"""Start a simple HTTP server in a background thread and yield its URL."""
|
||||
# Define the handler to serve files from the current directory
|
||||
handler = http.server.SimpleHTTPRequestHandler
|
||||
def simple_file_server(test_directory):
|
||||
# Use a random port for testing
|
||||
server_address = ('', 0) # 0 means pick any available port
|
||||
|
||||
# Set up an HTTP server on a free port
|
||||
with socketserver.TCPServer(("localhost", 0), handler) as httpd:
|
||||
# Get the port where the server is running
|
||||
port = httpd.server_address[1]
|
||||
server_url = f"http://localhost:{port}/"
|
||||
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
# Pass the directory to the handler
|
||||
self.directory = kwargs['directory']
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# Start the server in a background thread
|
||||
server_thread = threading.Thread(target=httpd.serve_forever)
|
||||
server_thread.daemon = True
|
||||
server_thread.start()
|
||||
# Create the server
|
||||
server = http.server.HTTPServer(
|
||||
server_address,
|
||||
# Hack to pass `test_directory` as `self.directory` to `SimpleHTTPRequestHandler`
|
||||
lambda *args, **kwargs:
|
||||
CustomHTTPRequestHandler(*args, directory=str(test_directory), **kwargs))
|
||||
|
||||
# Yield the URL of the running server
|
||||
try:
|
||||
yield httpd, server_url
|
||||
finally:
|
||||
# Stop the server after the test
|
||||
httpd.shutdown()
|
||||
server_thread.join()
|
||||
# Start the server in a background thread
|
||||
server_thread = threading.Thread(target=server.serve_forever)
|
||||
server_thread.daemon = True # So the server dies when the main thread dies
|
||||
server_thread.start()
|
||||
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
# Cleanup
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
server_thread.join()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_downloader_download_file():
|
||||
"""Mock the Downloader.download_file method."""
|
||||
contents = None
|
||||
|
||||
async def _download_file(self, id_, url, destination):
|
||||
name = url.split('/')[-1]
|
||||
output_path = destination / name
|
||||
output_path.write_bytes(contents or b'')
|
||||
return output_path
|
||||
|
||||
with mock.patch('wrolpi.downloader.Downloader.download_file', _download_file):
|
||||
def set_contents(contents_):
|
||||
nonlocal contents
|
||||
contents = contents_
|
||||
|
||||
yield set_contents
|
||||
|
|
|
|||
|
|
@ -175,3 +175,15 @@ class RefreshConflict(APIError):
|
|||
code = 'REFRESH_CONFLICT'
|
||||
message = 'Not possible during file refresh'
|
||||
status_code = HTTPStatus.CONFLICT
|
||||
|
||||
|
||||
class IgnoredDirectoryError(APIError):
|
||||
code = 'IGNORED_DIRECTORY_ERROR'
|
||||
message = 'Directory is being ignored'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class DownloadError(APIError):
|
||||
code = 'DOWNLOAD_ERROR'
|
||||
message = 'Unable to complete download'
|
||||
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
|
|
|
|||
120
wrolpi/status.py
120
wrolpi/status.py
|
|
@ -31,6 +31,7 @@ UPTIME_BIN = which('uptime', '/usr/bin/uptime')
|
|||
warn_once = get_warn_once('Unable to use psutil', logger)
|
||||
warn_cpu_once = get_warn_once('Cannot read CPU stats', logger)
|
||||
warn_cpu_temperature_once = get_warn_once('Cannot read CPU Temperature stats', logger)
|
||||
warn_iostat_once = get_warn_once('Cannot get iostat stats', logger)
|
||||
status_worker_warn_once = get_warn_once('Status worker encountered error', logger)
|
||||
|
||||
|
||||
|
|
@ -62,6 +63,58 @@ async def get_load_stats() -> SystemLoad:
|
|||
return load
|
||||
|
||||
|
||||
@dataclass
|
||||
class IostatInfo:
|
||||
percent_idle: float = None
|
||||
percent_iowait: float = None
|
||||
percent_nice: float = None
|
||||
percent_steal: float = None
|
||||
percent_system: float = None
|
||||
percent_user: float = None
|
||||
|
||||
def __json__(self) -> dict:
|
||||
return dict(
|
||||
percent_idle=self.percent_idle,
|
||||
percent_iowait=self.percent_iowait,
|
||||
percent_nice=self.percent_nice,
|
||||
percent_steal=self.percent_steal,
|
||||
percent_system=self.percent_system,
|
||||
percent_user=self.percent_user,
|
||||
)
|
||||
|
||||
|
||||
async def get_iostat_stats() -> IostatInfo:
|
||||
logger.trace(f'get_iostat_stats called')
|
||||
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
'iostat',
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if stdout and proc.returncode == 0:
|
||||
iostat_stats = jc.parse('iostat', stdout.decode(), quiet=True)
|
||||
if iostat_stats:
|
||||
cpu_stats, *_ = iostat_stats
|
||||
|
||||
logger.debug(
|
||||
f'get_iostat_stats got percent_iowait={cpu_stats["percent_iowait"]} percent_idle={cpu_stats["percent_idle"]}')
|
||||
# Return only the CPU stats.
|
||||
return IostatInfo(
|
||||
percent_idle=cpu_stats['percent_idle'],
|
||||
percent_iowait=cpu_stats['percent_iowait'],
|
||||
percent_nice=cpu_stats['percent_nice'],
|
||||
percent_steal=cpu_stats['percent_steal'],
|
||||
percent_system=cpu_stats['percent_system'],
|
||||
percent_user=cpu_stats['percent_user'],
|
||||
)
|
||||
except Exception as e:
|
||||
warn_iostat_once(e)
|
||||
|
||||
return IostatInfo()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessInfo:
|
||||
pid: int = None
|
||||
|
|
@ -113,16 +166,9 @@ async def get_cpu_stats() -> CPUInfo:
|
|||
"""Get core count, max freq, min freq, current freq, cpu temperature."""
|
||||
percent = int(psutil.cpu_percent())
|
||||
|
||||
# Get temperature from system file.
|
||||
temperature = cur_frequency = max_frequency = min_frequency = None
|
||||
if TEMPERATURE_PATH.is_file():
|
||||
try:
|
||||
temperature = int(TEMPERATURE_PATH.read_text()) // 1000
|
||||
min_frequency = int(MIN_FREQUENCY_PATH.read_text())
|
||||
max_frequency = int(MAX_FREQUENCY_PATH.read_text())
|
||||
cur_frequency = int(CUR_FREQUENCY_PATH.read_text())
|
||||
except Exception as e:
|
||||
warn_cpu_once(e)
|
||||
min_frequency = int(MIN_FREQUENCY_PATH.read_text())
|
||||
max_frequency = int(MAX_FREQUENCY_PATH.read_text())
|
||||
cur_frequency = int(CUR_FREQUENCY_PATH.read_text())
|
||||
|
||||
# Get temperature using psutil.
|
||||
temperature = high_temperature = critical_temperature = None
|
||||
|
|
@ -480,6 +526,41 @@ async def get_disk_counters(shared_status):
|
|||
return disk_bandwidth_stats
|
||||
|
||||
|
||||
@dataclass
|
||||
class PowerStats:
|
||||
under_voltage: bool = False
|
||||
over_current: bool = False
|
||||
|
||||
def __json__(self) -> dict:
|
||||
return dict(
|
||||
under_voltage=self.under_voltage,
|
||||
over_current=self.over_current,
|
||||
)
|
||||
|
||||
|
||||
async def get_power_stats() -> PowerStats:
|
||||
logger.trace(f'get_power_stats called')
|
||||
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
'dmesg',
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if stdout and proc.returncode == 0:
|
||||
under_voltage = b' Undervoltage detected' in stdout
|
||||
over_current = b' over-current change ' in stdout
|
||||
return PowerStats(
|
||||
under_voltage=under_voltage,
|
||||
over_current=over_current,
|
||||
)
|
||||
except Exception as e:
|
||||
warn_iostat_once(e)
|
||||
|
||||
return PowerStats()
|
||||
|
||||
|
||||
@perpetual_signal(sleep=5)
|
||||
async def status_worker(count: int = None, sleep_time: int = 5):
|
||||
"""A background process which will gather historical data about system statistics."""
|
||||
|
|
@ -492,13 +573,16 @@ async def status_worker(count: int = None, sleep_time: int = 5):
|
|||
load_stats = None
|
||||
try:
|
||||
# Update global `status` dict with stats that are gathered instantly.
|
||||
cpu_stats, load_stats, drives_stats, memory_stats, processes_stats = await asyncio.gather(
|
||||
get_cpu_stats(),
|
||||
get_load_stats(),
|
||||
get_drives_stats(),
|
||||
get_memory_stats(),
|
||||
get_processes_stats(),
|
||||
)
|
||||
cpu_stats, load_stats, drives_stats, memory_stats, processes_stats, iostat_stats, power_stats = \
|
||||
await asyncio.gather(
|
||||
get_cpu_stats(),
|
||||
get_load_stats(),
|
||||
get_drives_stats(),
|
||||
get_memory_stats(),
|
||||
get_processes_stats(),
|
||||
get_iostat_stats(),
|
||||
get_power_stats(),
|
||||
)
|
||||
shared_status.update({
|
||||
'cpu_stats': cpu_stats.__json__(),
|
||||
'load_stats': load_stats.__json__(),
|
||||
|
|
@ -506,6 +590,8 @@ async def status_worker(count: int = None, sleep_time: int = 5):
|
|||
'processes_stats': [i.__json__() for i in processes_stats],
|
||||
'memory_stats': memory_stats.__json__(),
|
||||
'last_status': now().isoformat(),
|
||||
'iostat_stats': iostat_stats.__json__(),
|
||||
'power_stats': power_stats.__json__(),
|
||||
})
|
||||
|
||||
if 'disk_bandwidth_stats' not in shared_status:
|
||||
|
|
|
|||
|
|
@ -146,4 +146,4 @@ async def await_switches(timeout: int = 10):
|
|||
# All switches handled.
|
||||
break
|
||||
else:
|
||||
raise RuntimeError('Timed out waiting for switches')
|
||||
raise RuntimeError('Timed out waiting for switches. Did you remember to use the `await_switches` fixture?')
|
||||
|
|
|
|||
|
|
@ -834,24 +834,22 @@ def test_config_backup(async_client, test_wrolpi_config, test_directory):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@skip_circleci
|
||||
async def test_aiohttp(simple_web_server):
|
||||
async def test_aiohttp(simple_file_server):
|
||||
"""Test that aiohttp convenience functions work."""
|
||||
httpd, server_url = simple_web_server
|
||||
host, port = simple_file_server.server_address
|
||||
url = f'http://{host}:{port}' # noqa
|
||||
|
||||
async with common.aiohttp_get(server_url) as response:
|
||||
async with common.aiohttp_get(url) as response:
|
||||
assert response.content.read()
|
||||
assert response.status == 200
|
||||
|
||||
async with common.aiohttp_head(server_url) as response:
|
||||
async with common.aiohttp_head(url) as response:
|
||||
assert response.content.read()
|
||||
assert response.status == 200
|
||||
|
||||
|
||||
def test_can_connect_to_server(simple_web_server):
|
||||
httpd, server_url = simple_web_server
|
||||
_, host, port = server_url.split(':')
|
||||
host = host[2:]
|
||||
port = int(port[:-1])
|
||||
def test_can_connect_to_server(simple_file_server):
|
||||
host, port = simple_file_server.server_address
|
||||
|
||||
assert common.can_connect_to_server(host, port)
|
||||
|
||||
|
|
@ -1121,28 +1119,28 @@ def test_get_paths_in_parent_directory(test_directory, make_files_structure):
|
|||
|
||||
paths = list(common.walk(test_directory))
|
||||
|
||||
assert common.get_paths_in_parent_directory(paths, test_directory / 'foo') == [
|
||||
assert set(common.get_paths_in_parent_directory(paths, test_directory / 'foo')) == {
|
||||
test_directory / 'foo/bar.txt',
|
||||
test_directory / 'foo/bar',
|
||||
test_directory / 'foo/bar/baz.txt',
|
||||
]
|
||||
}
|
||||
|
||||
assert common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar') == [
|
||||
assert set(common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar')) == {
|
||||
test_directory / 'foo/bar/baz.txt',
|
||||
]
|
||||
}
|
||||
|
||||
assert common.get_paths_in_parent_directory(paths, test_directory) == [
|
||||
assert set(common.get_paths_in_parent_directory(paths, test_directory)) == {
|
||||
test_directory / 'qux.txt',
|
||||
test_directory / 'foo',
|
||||
test_directory / 'foo/bar.txt',
|
||||
test_directory / 'foo/bar',
|
||||
test_directory / 'foo/bar/baz.txt',
|
||||
]
|
||||
}
|
||||
|
||||
assert common.get_paths_in_media_directory(paths) == [
|
||||
assert set(common.get_paths_in_media_directory(paths)) == {
|
||||
test_directory / 'qux.txt',
|
||||
test_directory / 'foo',
|
||||
test_directory / 'foo/bar.txt',
|
||||
test_directory / 'foo/bar',
|
||||
test_directory / 'foo/bar/baz.txt',
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,21 @@ async def test_get_cpu_stats(async_client):
|
|||
assert isinstance(info.temperature, int)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iostat_stats(async_client):
|
||||
info = await status.get_iostat_stats()
|
||||
assert isinstance(info, status.IostatInfo)
|
||||
assert isinstance(info.percent_idle, float)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_power_stats(async_client):
|
||||
info = await status.get_power_stats()
|
||||
assert isinstance(info, status.PowerStats)
|
||||
assert info.under_voltage is False
|
||||
assert info.over_current is False
|
||||
|
||||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_drives_stats(async_client):
|
||||
|
|
@ -57,3 +72,11 @@ async def test_status_worker(async_client):
|
|||
assert 'bytes_recv' in list(api_app.shared_ctx.status['nic_bandwidth_stats'].values())[0]
|
||||
assert api_app.shared_ctx.status['disk_bandwidth_stats']
|
||||
assert 'bytes_read' in list(api_app.shared_ctx.status['disk_bandwidth_stats'].values())[0]
|
||||
|
||||
assert isinstance(api_app.shared_ctx.status.get('cpu_stats'), dict)
|
||||
assert isinstance(api_app.shared_ctx.status.get('load_stats'), dict)
|
||||
assert isinstance(api_app.shared_ctx.status.get('drives_stats'), list)
|
||||
assert isinstance(api_app.shared_ctx.status.get('processes_stats'), list)
|
||||
assert isinstance(api_app.shared_ctx.status.get('memory_stats'), dict)
|
||||
assert isinstance(api_app.shared_ctx.status.get('iostat_stats'), dict)
|
||||
assert isinstance(api_app.shared_ctx.status.get('power_stats'), dict)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue