Using iostat to detect busy disk issues, reporting them in the UI. Detecting under-voltage/over-current issues using dmesg and reporting them in the UI.

This commit is contained in:
Roland 2025-01-10 11:33:21 -07:00
parent 220cb8010f
commit d8b3db9d63
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
14 changed files with 339 additions and 282 deletions

View file

@ -20,15 +20,7 @@ import {
} from 'semantic-ui-react';
import {Link, NavLink, useNavigate, useSearchParams} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {
useCPUTemperature,
useHotspot,
useLoad,
useSearchDirectories,
useSearchOrder,
useThrottle,
useWROLMode
} from "../hooks/customHooks";
import {useHotspot, useSearchDirectories, useSearchOrder, useThrottle, useWROLMode} from "../hooks/customHooks";
import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts";
import {
Accordion,
@ -1124,38 +1116,6 @@ export function HotspotStatusIcon() {
</>
}
export function CPUTemperatureIcon({size = 'large', fallback = null}) {
// Returns an Icon only if temperature is too high.
const {temperature, highTemperature, criticalTemperature} = useCPUTemperature();
if (temperature === null || temperature < highTemperature) {
// Temperature is not a problem.
return fallback;
}
let icon = temperature >= criticalTemperature ?
<Icon data-testid='cpuTemperatureIcon' name='thermometer' size={size} color='red'/>
: <Icon data-testid='cpuTemperatureIcon' name='thermometer half' size={size} color='yellow'/>;
icon = <Link to='/admin/status'>{icon}</Link>;
return <Popup content={`CPU: ${temperature}°C`} trigger={icon}/>
}
export function SystemLoadIcon({size = 'large', fallback = null}) {
const {minute_1, mediumLoad, highLoad} = useLoad();
if (!mediumLoad && !highLoad) {
return fallback;
}
// Only return the icon if the load is not low.
const color = highLoad ? 'red' : 'yellow';
let icon = <Link to='/admin/status'>
<Icon name='tachometer alternate' size={size} color={color}/>
</Link>;
return <Popup content={`Load: ${minute_1}`} trigger={icon}/>
}
export function useTitle(title) {
const documentDefined = typeof document !== 'undefined';
const originalTitle = React.useRef(documentDefined ? document.title : null);
@ -1950,3 +1910,11 @@ export function getDistinctColor(hexColors) {
// If we've tried maxAttempts times, return the last generated color regardless
return hslToHex(newColor.h, newColor.s, newColor.l);
}
export const isReactChildNull = (children) => {
// Returns `true` if a React element returns `null` as it's child.
const nonNullChildren = React.Children.map(children,
(child) => React.isValidElement(child) && child.type(child.props)
);
return nonNullChildren.length === 0;
}

View file

@ -99,7 +99,7 @@ const ViewerMessage = () => {
function SlowImportMessage() {
const {status} = useContext(StatusContext);
if (status && status['cpu_info'] && status['cpu_info']['temperature'] >= 80) {
if (status && status['cpu_stats'] && status['cpu_stats']['temperature'] >= 80) {
return <Message warning icon>
<SIcon name='exclamation'/>
<Message.Content>

View file

@ -2,11 +2,11 @@ import React from "react";
import {Link, NavLink} from "react-router-dom";
import {Dropdown, Icon as SIcon, Menu} from "semantic-ui-react";
import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts";
import {CPUTemperatureIcon, DarkModeToggle, HotspotStatusIcon, SystemLoadIcon, useLocalStorage} from "./Common";
import {DarkModeToggle, HotspotStatusIcon, isReactChildNull, useLocalStorage} from "./Common";
import {ShareButton} from "./Share";
import {useWROLMode} from "../hooks/customHooks";
import {useCPUTemperature, useIOStats, useLoad, useMemoryStats, usePowerStats, useWROLMode} from "../hooks/customHooks";
import {SearchIconButton} from "./Search";
import {Icon} from "./Theme";
import {Icon, Popup} from "./Theme";
import {HELP_VIEWER_URI, NAME} from "./Vars";
import _ from "lodash";
@ -59,15 +59,11 @@ function MenuLink({link}) {
}
}
function NavIconWrapper(props) {
if (_.isEmpty(props.children)) {
return <></>
} else {
if (props.name === 'warning') {
// console.log(props.name, props.children);
}
return <div style={{marginTop: '0.8em', marginLeft: '1.5em'}}>{props.children}</div>
}
function NavIconWrapper({children}) {
// Do not add margins around empty react element to avoid filling up navbar.
const emptyChild = !children || isReactChildNull(children);
const style = emptyChild ? {} : {marginTop: '0.8em', marginLeft: '1.5em'};
return <div style={style}>{children}</div>
}
function useNavColorSetting() {
@ -97,8 +93,74 @@ export function NavBar() {
{topNavText}
</NavLink>;
// Display the temperature icon first because it can cause the system to throttle.
const warningIcon = <CPUTemperatureIcon fallback={<SystemLoadIcon/>}/>;
// Red/Yellow colors will blend in to some navbar colors, swap the colors for high contrast.
const conflictingColors = ['red', 'orange', 'yellow', 'olive', 'pink'];
const lowWarningColor = conflictingColors.includes(navColor) ? 'white' : 'yellow';
const highWarningColor = conflictingColors.includes(navColor) ? 'black' : 'red';
// Generic system load, the least important warning.
const {minute_1, mediumLoad, highLoad} = useLoad();
let systemLoadIcon;
if (mediumLoad || highLoad) {
// System load is high, display a warning icon.
const color = highLoad ? highWarningColor : lowWarningColor;
const icon = <Link to='/admin/status'>
<Icon name='tachometer alternate' size='large' color={color}/>
</Link>;
systemLoadIcon = <Popup content={`Load: ${minute_1}`} trigger={icon}/>
}
// RAM consumption.
const {percent: memoryPercent} = useMemoryStats();
let memoryIcon;
if (memoryPercent > 80) {
const color = memoryPercent > 90 ? highWarningColor : lowWarningColor;
const icon = <Link to='/admin/status' color={color}>
<Icon name='microchip' size='large'/>
</Link>;
memoryIcon = <Popup content={`System Memory: ${memoryPercent}%`} trigger={icon}/>
}
// Any disk is busy and processes are waiting.
const {percentIOWait} = useIOStats();
let diskWaitIcon;
if (percentIOWait >= 50) {
// Processes are waiting on disk, display a warning icon.
const color = percentIOWait > 75 ? highWarningColor : lowWarningColor;
const icon = <Link to='/admin/status' color={color}>
<Icon name='disk' size='large'/>
</Link>;
diskWaitIcon = <Popup content={`Processes waiting on disk: ${percentIOWait}`} trigger={icon}/>
}
// CPU temperature.
const {temperature, highTemperature, criticalTemperature} = useCPUTemperature();
let temperatureIcon;
if (temperature && temperature >= highTemperature) {
// CPU temperature is high, display a warning icon.
const color = temperature >= criticalTemperature ? highWarningColor : lowWarningColor;
const name = temperature >= criticalTemperature ? 'thermometer' : 'thermometer half';
const icon = <Icon data-testid='cpuTemperatureIcon' name={name} size='large' color={color}/>
const link = <Link to='/admin/status'>{icon}</Link>;
temperatureIcon = <Popup content={`CPU: ${temperature}°C`} trigger={link}/>;
}
// Power issues, this is always displayed if detected.
const {underVoltage, overCurrent} = usePowerStats();
let powerIcon;
if (underVoltage || overCurrent) {
const name = underVoltage ? 'power cord' : 'lightning';
const icon = <Icon name={name} size='large' color={highWarningColor}/>;
const message = underVoltage
? 'Under-voltage detected! Your power supply is insufficient!'
: 'Over-current detected! Your peripherals are using too much power!';
powerIcon = <Popup content={message} trigger={icon}/>;
}
// Display the temperature icon first because it can cause the system to throttle. The rest are in order of effects
// that will slow down the system and the user should address. Generic load is last because it is probably not an
// issue.
const warningIcon = temperatureIcon || diskWaitIcon || memoryIcon || systemLoadIcon;
let processingLink;
if (status && status.flags) {
@ -115,11 +177,12 @@ export function NavBar() {
</Link>;
const icons = <React.Fragment>
<NavIconWrapper name='processing'>{processingIcon}</NavIconWrapper>
<NavIconWrapper name='warning'>{warningIcon}</NavIconWrapper>
<NavIconWrapper name='share'><ShareButton/></NavIconWrapper>
<NavIconWrapper name='hotspot'><HotspotStatusIcon/></NavIconWrapper>
<NavIconWrapper name='dark mode'><DarkModeToggle/></NavIconWrapper>
<NavIconWrapper>{processingIcon}</NavIconWrapper>
<NavIconWrapper>{powerIcon}</NavIconWrapper>
<NavIconWrapper>{warningIcon}</NavIconWrapper>
<NavIconWrapper><ShareButton/></NavIconWrapper>
<NavIconWrapper><HotspotStatusIcon/></NavIconWrapper>
<NavIconWrapper><DarkModeToggle/></NavIconWrapper>
</React.Fragment>;
return <>

View file

@ -877,39 +877,29 @@ export const StatusProvider = (props) => {
export const useStatusFlag = (flag) => {
const {status} = useContext(StatusContext);
return status && status['flags'] && status['flags'][flag];
return status?.flags?.[flag] || false;
}
export const useCPUTemperature = () => {
const {status} = React.useContext(StatusContext);
let temperature = 0;
let highTemperature = 75;
let criticalTemperature = 85;
if (status && status['cpu_info']) {
temperature = status['cpu_info']['temperature'];
highTemperature = status['cpu_info']['high_temperature'];
criticalTemperature = status['cpu_info']['critical_temperature'];
}
const temperature = status?.cpu_stats?.temperature || 0;
const highTemperature = status?.cpu_stats?.high_temperature || 75;
const criticalTemperature = status?.cpu_stats?.critical_temperature || 85;
return {temperature, highTemperature, criticalTemperature}
}
export const useLoad = () => {
const {status} = React.useContext(StatusContext);
let minute_1;
let minute_5;
let minute_15;
let minute_1 = status?.load_stats?.minute_1;
let minute_5 = status?.load_stats?.minute_5;
let minute_15 = status?.load_stats?.minute_15;
let mediumLoad = false;
let highLoad = false;
let cores;
if (status && status['load']) {
minute_1 = status['load']['minute_1'];
minute_5 = status['load']['minute_5'];
minute_15 = status['load']['minute_15'];
cores = status['cpu_info']['cores'];
if (status && status.cpu_stats) {
// Medium load when 1/2 cores are busy, High load when 3/4+ cores are busy.
cores = status.cpu_stats.cores;
const quarter = cores / 4;
if (cores && minute_1 >= (quarter * 3)) {
highLoad = true;
@ -921,6 +911,37 @@ export const useLoad = () => {
return {minute_1, minute_5, minute_15, mediumLoad, highLoad, cores};
}
export const useIOStats = () => {
const {status} = React.useContext(StatusContext);
const percentIdle = status?.iostat_stats?.percent_idle;
const percentIOWait = status?.iostat_stats?.percent_iowait;
const percentNice = status?.iostat_stats?.percent_nice;
const percentSteal = status?.iostat_stats?.percent_steal;
const percentSystem = status?.iostat_stats?.percent_system;
const percentUser = status?.iostat_stats?.percent_user;
return {percentIdle, percentIOWait, percentNice, percentSteal, percentSystem, percentUser};
}
export const useMemoryStats = () => {
const {status} = React.useContext(StatusContext);
const total = status?.memory_stats?.total;
const used = status?.memory_stats?.used;
const free = status?.memory_stats?.free;
const cached = status?.memory_stats?.cached;
let percent;
if (total && used) {
percent = Math.round((used / total) * 100);
}
return {total, used, free, cached, percent}
}
export const usePowerStats = () => {
const {status} = React.useContext(StatusContext);
const underVoltage = status?.power_stats?.under_voltage;
const overCurrent = status?.power_stats?.over_current;
return {underVoltage, overCurrent};
}
export const useVideoStatistics = () => {
const [statistics, setStatistics] = useState(null);

View file

@ -91,6 +91,7 @@ rsync
smartmontools
software-properties-common
sudo
sysstat
tar
tcpdump
tmux

View file

@ -5,7 +5,7 @@ WORKDIR /opt/wrolpi
# Update dependencies for the services install.
RUN apt update
# ffmpeg for Videos
RUN apt-get install -y ffmpeg catdoc
RUN apt-get install -y ffmpeg catdoc sysstat
RUN ffmpeg -version
# yt-dlp needs cache directory
RUN mkdir /.cache && chown -R 1000:1000 /.cache

View file

@ -88,6 +88,7 @@ renderd
rsync
smartmontools
software-properties-common
sysstat
tar
tcpdump
tmux

View file

@ -21,7 +21,6 @@ from dataclasses import dataclass, asdict, field, fields
from datetime import datetime, date
from decimal import Decimal
from functools import wraps
from http import HTTPStatus
from itertools import islice, filterfalse, tee
from multiprocessing.managers import DictProxy
from pathlib import Path
@ -40,8 +39,8 @@ from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError, InvalidConfig, \
from wrolpi.dates import now, from_timestamp
from wrolpi.errors import WROLModeEnabled, NativeOnly, LogLevelError, InvalidConfig, \
ValidationError
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS
@ -240,6 +239,7 @@ __all__ = [
'apply_modelers',
'apply_refresh_cleanup',
'background_task',
'can_connect_to_server',
'cancel_background_tasks',
'cancel_refresh_tasks',
'cancelable_wrapper',
@ -249,10 +249,10 @@ __all__ = [
'chunks',
'chunks_by_stem',
'compile_tsvector',
'create_empty_config_files',
'cum_timer',
'date_range',
'disable_wrol_mode',
'download_file',
'enable_wrol_mode',
'escape_file_name',
'extract_domain',
@ -1414,48 +1414,6 @@ async def aiohttp_head(url: str, timeout: int = None, headers: dict = None) -> C
yield response
async def speed_test(url: str, timeout: int | None = 10) -> int:
"""Request the last megabyte of the provided url, return the content length divided by the time elapsed (speed)."""
async with aiohttp_head(url) as response:
content_length = response.headers['Content-Length']
start_bytes = int(content_length) - 10485760
range_ = f'bytes={start_bytes}-{content_length}'
start_time = datetime.now()
async with aiohttp_get(url, headers={'Range': range_}, timeout=timeout) as response:
content = await response.content.read()
elapsed = int((datetime.now() - start_time).total_seconds())
return len(content) // elapsed
async def get_fastest_mirror(urls: List[str]) -> str:
"""Perform a speed test on each URL, return the fastest."""
fastest_url = urls[0]
fastest_speed = 0
for url in urls:
try:
# Timeout after 5 minutes. Mirror is less than 27 kbps.
speed = await speed_test(url, timeout=60 * 5)
download_logger.debug(f'Speed test of {url} was {speed} bps')
except Exception as e:
download_logger.error(f'Speedtest of {url} failed', exc_info=e)
continue
if speed > fastest_speed:
fastest_url = url
return fastest_url
@dataclass
class DownloadFileInfoLink:
"""Represents an HTTP "Link" Header."""
url: str
rel: str
type: str
priority: int
geo: str
@dataclass
class DownloadFileInfo:
"""Information about a file that can be downloaded."""
@ -1465,7 +1423,6 @@ class DownloadFileInfo:
accept_ranges: str = None
status: int = None
location: str = None
links: List[DownloadFileInfoLink] = None
FILENAME_MATCHER = re.compile(r'.*filename="(.*)"')
@ -1475,28 +1432,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
"""Gets information (name, size, etc.) about a downloadable file at the provided URL."""
async with aiohttp_head(url, timeout) as response:
download_logger.debug(f'{response.headers=}')
try:
links = response.headers.getall('Link')
except KeyError:
links = None
new_links = list()
if links:
# Convert "Link" header strings to DownloadFileInfoLink.
for idx, link in enumerate(links):
url, *props = link.split(';')
url = url[1:-1]
properties = dict()
for prop in props:
name, value = prop.strip().split('=')
properties[name] = value
new_links.append(DownloadFileInfoLink(
url,
properties.get('rel').strip() if 'rel' in properties else None,
properties.get('type').strip() if 'type' in properties else None,
int(properties.get('pri').strip()) if 'pri' in properties else None,
properties.get('geo').strip() if 'geo' in properties else None,
))
info = DownloadFileInfo(
type=response.headers.get('Content-Type'),
@ -1504,7 +1439,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
accept_ranges=response.headers.get('Accept-Ranges'),
status=response.status,
location=response.headers.get('Location'),
links=new_links,
)
disposition = response.headers.get('Content-Disposition')
@ -1526,80 +1460,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
download_logger = logger.getChild('download')
async def download_file(url: str, output_path: pathlib.Path = None, info: DownloadFileInfo = None,
timeout: int = 7 * 24 * 60 * 60):
"""Uses aiohttp to download an HTTP file. Performs a speed test when mirrors are found, downloads from the fastest
mirror.
Attempts to resume the file if `output_path` already exists.
@warning: Timeout default is a week because of large downloads.
"""
info = info or await get_download_info(url, timeout)
if output_path.is_file() and info.size == output_path.stat().st_size:
download_logger.warning(f'Already downloaded {repr(str(url))} to {repr(str(output_path))}')
return
if info.links and (mirror_urls := [i.url for i in info.links if i.rel == 'duplicate']):
# Mirrors are available, find the fastest.
download_logger.info(f'Performing download speed test on mirrors: {mirror_urls}')
url = await get_fastest_mirror(mirror_urls)
info = await get_download_info(url, timeout)
download_logger.debug(f'Final DownloadInfo fetched {info}')
total_size = info.size
download_logger.info(f'Starting download of {url} with {total_size} total bytes')
if info.accept_ranges == 'bytes' or not output_path.is_file():
with output_path.open('ab') as fh:
headers = DEFAULT_HTTP_HEADERS.copy()
# Check the position of append, if it is 0 then we do not need to resume.
position = fh.tell()
if position:
headers['Range'] = f'bytes={position}-'
async with aiohttp_session(timeout=timeout) as session:
async with session.get(url, headers=headers) as response:
response: ClientResponse
if response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
download_logger.warning(f'Server responded with 416, file is probably already downloaded')
return
if position and response.status != HTTPStatus.PARTIAL_CONTENT:
raise UnrecoverableDownloadError(
f'Tried to resume {repr(str(url))} but got status {response.status}')
# May or may not be using Range. Append each chunk to the output file.
last_report = datetime.now()
bytes_received = 0
async for data in response.content.iter_any():
fh.write(data)
bytes_received += len(data)
if (elapsed := (datetime.now() - last_report).total_seconds()) > 10:
# Report download speed every 10 seconds.
bytes_per_second = int(bytes_received // elapsed)
download_logger.debug(f'{bytes_received=} {elapsed=} {bytes_per_second=}')
size = fh.tell()
bytes_remaining = total_size - size
seconds_remaining = bytes_remaining // bytes_per_second
percent = int((size / total_size) * 100)
download_logger.info(
f'Downloading {url} at'
f' rate={human_bandwidth(bytes_per_second)}'
f' estimate={seconds_to_timestamp(seconds_remaining)}')
download_logger.debug(f'Downloading {url} {total_size=} {size=} {percent=}')
last_report = datetime.now()
bytes_received = 0
# Sleep to catch cancel.
await asyncio.sleep(0)
elif output_path.is_file():
# TODO support downloading files that cannot be resumed.
raise UnrecoverableDownloadError(f'Cannot resume download {url}')
def human_bandwidth(bps: int) -> str:
"""Convert bits per second to a more readable format.

View file

@ -9,7 +9,6 @@ import logging
import multiprocessing
import pathlib
import shutil
import socketserver
import tempfile
import threading
import zipfile
@ -760,26 +759,51 @@ def make_multipart_form():
@pytest.fixture
def simple_web_server():
"""Start a simple HTTP server in a background thread and yield its URL."""
# Define the handler to serve files from the current directory
handler = http.server.SimpleHTTPRequestHandler
def simple_file_server(test_directory):
# Use a random port for testing
server_address = ('', 0) # 0 means pick any available port
# Set up an HTTP server on a free port
with socketserver.TCPServer(("localhost", 0), handler) as httpd:
# Get the port where the server is running
port = httpd.server_address[1]
server_url = f"http://localhost:{port}/"
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
# Pass the directory to the handler
self.directory = kwargs['directory']
super().__init__(*args, **kwargs)
# Start the server in a background thread
server_thread = threading.Thread(target=httpd.serve_forever)
server_thread.daemon = True
server_thread.start()
# Create the server
server = http.server.HTTPServer(
server_address,
# Hack to pass `test_directory` as `self.directory` to `SimpleHTTPRequestHandler`
lambda *args, **kwargs:
CustomHTTPRequestHandler(*args, directory=str(test_directory), **kwargs))
# Yield the URL of the running server
try:
yield httpd, server_url
finally:
# Stop the server after the test
httpd.shutdown()
server_thread.join()
# Start the server in a background thread
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True # So the server dies when the main thread dies
server_thread.start()
try:
yield server
finally:
# Cleanup
server.shutdown()
server.server_close()
server_thread.join()
@pytest.fixture
def mock_downloader_download_file():
"""Mock the Downloader.download_file method."""
contents = None
async def _download_file(self, id_, url, destination):
name = url.split('/')[-1]
output_path = destination / name
output_path.write_bytes(contents or b'')
return output_path
with mock.patch('wrolpi.downloader.Downloader.download_file', _download_file):
def set_contents(contents_):
nonlocal contents
contents = contents_
yield set_contents

View file

@ -175,3 +175,15 @@ class RefreshConflict(APIError):
code = 'REFRESH_CONFLICT'
message = 'Not possible during file refresh'
status_code = HTTPStatus.CONFLICT
class IgnoredDirectoryError(APIError):
code = 'IGNORED_DIRECTORY_ERROR'
message = 'Directory is being ignored'
status_code = HTTPStatus.BAD_REQUEST
class DownloadError(APIError):
code = 'DOWNLOAD_ERROR'
message = 'Unable to complete download'
status_code = HTTPStatus.INTERNAL_SERVER_ERROR

View file

@ -31,6 +31,7 @@ UPTIME_BIN = which('uptime', '/usr/bin/uptime')
warn_once = get_warn_once('Unable to use psutil', logger)
warn_cpu_once = get_warn_once('Cannot read CPU stats', logger)
warn_cpu_temperature_once = get_warn_once('Cannot read CPU Temperature stats', logger)
warn_iostat_once = get_warn_once('Cannot get iostat stats', logger)
status_worker_warn_once = get_warn_once('Status worker encountered error', logger)
@ -62,6 +63,58 @@ async def get_load_stats() -> SystemLoad:
return load
@dataclass
class IostatInfo:
percent_idle: float = None
percent_iowait: float = None
percent_nice: float = None
percent_steal: float = None
percent_system: float = None
percent_user: float = None
def __json__(self) -> dict:
return dict(
percent_idle=self.percent_idle,
percent_iowait=self.percent_iowait,
percent_nice=self.percent_nice,
percent_steal=self.percent_steal,
percent_system=self.percent_system,
percent_user=self.percent_user,
)
async def get_iostat_stats() -> IostatInfo:
logger.trace(f'get_iostat_stats called')
try:
proc = await asyncio.create_subprocess_shell(
'iostat',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if stdout and proc.returncode == 0:
iostat_stats = jc.parse('iostat', stdout.decode(), quiet=True)
if iostat_stats:
cpu_stats, *_ = iostat_stats
logger.debug(
f'get_iostat_stats got percent_iowait={cpu_stats["percent_iowait"]} percent_idle={cpu_stats["percent_idle"]}')
# Return only the CPU stats.
return IostatInfo(
percent_idle=cpu_stats['percent_idle'],
percent_iowait=cpu_stats['percent_iowait'],
percent_nice=cpu_stats['percent_nice'],
percent_steal=cpu_stats['percent_steal'],
percent_system=cpu_stats['percent_system'],
percent_user=cpu_stats['percent_user'],
)
except Exception as e:
warn_iostat_once(e)
return IostatInfo()
@dataclass
class ProcessInfo:
pid: int = None
@ -113,16 +166,9 @@ async def get_cpu_stats() -> CPUInfo:
"""Get core count, max freq, min freq, current freq, cpu temperature."""
percent = int(psutil.cpu_percent())
# Get temperature from system file.
temperature = cur_frequency = max_frequency = min_frequency = None
if TEMPERATURE_PATH.is_file():
try:
temperature = int(TEMPERATURE_PATH.read_text()) // 1000
min_frequency = int(MIN_FREQUENCY_PATH.read_text())
max_frequency = int(MAX_FREQUENCY_PATH.read_text())
cur_frequency = int(CUR_FREQUENCY_PATH.read_text())
except Exception as e:
warn_cpu_once(e)
min_frequency = int(MIN_FREQUENCY_PATH.read_text())
max_frequency = int(MAX_FREQUENCY_PATH.read_text())
cur_frequency = int(CUR_FREQUENCY_PATH.read_text())
# Get temperature using psutil.
temperature = high_temperature = critical_temperature = None
@ -480,6 +526,41 @@ async def get_disk_counters(shared_status):
return disk_bandwidth_stats
@dataclass
class PowerStats:
under_voltage: bool = False
over_current: bool = False
def __json__(self) -> dict:
return dict(
under_voltage=self.under_voltage,
over_current=self.over_current,
)
async def get_power_stats() -> PowerStats:
logger.trace(f'get_power_stats called')
try:
proc = await asyncio.create_subprocess_shell(
'dmesg',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if stdout and proc.returncode == 0:
under_voltage = b' Undervoltage detected' in stdout
over_current = b' over-current change ' in stdout
return PowerStats(
under_voltage=under_voltage,
over_current=over_current,
)
except Exception as e:
warn_iostat_once(e)
return PowerStats()
@perpetual_signal(sleep=5)
async def status_worker(count: int = None, sleep_time: int = 5):
"""A background process which will gather historical data about system statistics."""
@ -492,13 +573,16 @@ async def status_worker(count: int = None, sleep_time: int = 5):
load_stats = None
try:
# Update global `status` dict with stats that are gathered instantly.
cpu_stats, load_stats, drives_stats, memory_stats, processes_stats = await asyncio.gather(
get_cpu_stats(),
get_load_stats(),
get_drives_stats(),
get_memory_stats(),
get_processes_stats(),
)
cpu_stats, load_stats, drives_stats, memory_stats, processes_stats, iostat_stats, power_stats = \
await asyncio.gather(
get_cpu_stats(),
get_load_stats(),
get_drives_stats(),
get_memory_stats(),
get_processes_stats(),
get_iostat_stats(),
get_power_stats(),
)
shared_status.update({
'cpu_stats': cpu_stats.__json__(),
'load_stats': load_stats.__json__(),
@ -506,6 +590,8 @@ async def status_worker(count: int = None, sleep_time: int = 5):
'processes_stats': [i.__json__() for i in processes_stats],
'memory_stats': memory_stats.__json__(),
'last_status': now().isoformat(),
'iostat_stats': iostat_stats.__json__(),
'power_stats': power_stats.__json__(),
})
if 'disk_bandwidth_stats' not in shared_status:

View file

@ -146,4 +146,4 @@ async def await_switches(timeout: int = 10):
# All switches handled.
break
else:
raise RuntimeError('Timed out waiting for switches')
raise RuntimeError('Timed out waiting for switches. Did you remember to use the `await_switches` fixture?')

View file

@ -834,24 +834,22 @@ def test_config_backup(async_client, test_wrolpi_config, test_directory):
@pytest.mark.asyncio
@skip_circleci
async def test_aiohttp(simple_web_server):
async def test_aiohttp(simple_file_server):
"""Test that aiohttp convenience functions work."""
httpd, server_url = simple_web_server
host, port = simple_file_server.server_address
url = f'http://{host}:{port}' # noqa
async with common.aiohttp_get(server_url) as response:
async with common.aiohttp_get(url) as response:
assert response.content.read()
assert response.status == 200
async with common.aiohttp_head(server_url) as response:
async with common.aiohttp_head(url) as response:
assert response.content.read()
assert response.status == 200
def test_can_connect_to_server(simple_web_server):
httpd, server_url = simple_web_server
_, host, port = server_url.split(':')
host = host[2:]
port = int(port[:-1])
def test_can_connect_to_server(simple_file_server):
host, port = simple_file_server.server_address
assert common.can_connect_to_server(host, port)
@ -1121,28 +1119,28 @@ def test_get_paths_in_parent_directory(test_directory, make_files_structure):
paths = list(common.walk(test_directory))
assert common.get_paths_in_parent_directory(paths, test_directory / 'foo') == [
assert set(common.get_paths_in_parent_directory(paths, test_directory / 'foo')) == {
test_directory / 'foo/bar.txt',
test_directory / 'foo/bar',
test_directory / 'foo/bar/baz.txt',
]
}
assert common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar') == [
assert set(common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar')) == {
test_directory / 'foo/bar/baz.txt',
]
}
assert common.get_paths_in_parent_directory(paths, test_directory) == [
assert set(common.get_paths_in_parent_directory(paths, test_directory)) == {
test_directory / 'qux.txt',
test_directory / 'foo',
test_directory / 'foo/bar.txt',
test_directory / 'foo/bar',
test_directory / 'foo/bar/baz.txt',
]
}
assert common.get_paths_in_media_directory(paths) == [
assert set(common.get_paths_in_media_directory(paths)) == {
test_directory / 'qux.txt',
test_directory / 'foo',
test_directory / 'foo/bar.txt',
test_directory / 'foo/bar',
test_directory / 'foo/bar/baz.txt',
]
}

View file

@ -27,6 +27,21 @@ async def test_get_cpu_stats(async_client):
assert isinstance(info.temperature, int)
@pytest.mark.asyncio
async def test_iostat_stats(async_client):
info = await status.get_iostat_stats()
assert isinstance(info, status.IostatInfo)
assert isinstance(info.percent_idle, float)
@pytest.mark.asyncio
async def test_power_stats(async_client):
info = await status.get_power_stats()
assert isinstance(info, status.PowerStats)
assert info.under_voltage is False
assert info.over_current is False
@skip_circleci
@pytest.mark.asyncio
async def test_get_drives_stats(async_client):
@ -57,3 +72,11 @@ async def test_status_worker(async_client):
assert 'bytes_recv' in list(api_app.shared_ctx.status['nic_bandwidth_stats'].values())[0]
assert api_app.shared_ctx.status['disk_bandwidth_stats']
assert 'bytes_read' in list(api_app.shared_ctx.status['disk_bandwidth_stats'].values())[0]
assert isinstance(api_app.shared_ctx.status.get('cpu_stats'), dict)
assert isinstance(api_app.shared_ctx.status.get('load_stats'), dict)
assert isinstance(api_app.shared_ctx.status.get('drives_stats'), list)
assert isinstance(api_app.shared_ctx.status.get('processes_stats'), list)
assert isinstance(api_app.shared_ctx.status.get('memory_stats'), dict)
assert isinstance(api_app.shared_ctx.status.get('iostat_stats'), dict)
assert isinstance(api_app.shared_ctx.status.get('power_stats'), dict)