From d8b3db9d635c203958f7eed4be9522bee7f377be Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 10 Jan 2025 11:33:21 -0700 Subject: [PATCH] Using iostat to detect busy disk issues, reporting them in the UI. Detecting under-voltage/over-current issues using dmesg and reporting them in the UI. --- app/src/components/Common.js | 50 ++---- app/src/components/Map.js | 2 +- app/src/components/Nav.js | 101 +++++++++--- app/src/hooks/customHooks.js | 61 +++++--- .../config/package-lists/wrolpi.list.chroot | 1 + docker/api/Dockerfile | 2 +- pi-gen/stage2/04-wrolpi/01-packages | 1 + wrolpi/common.py | 148 +----------------- wrolpi/conftest.py | 66 +++++--- wrolpi/errors.py | 12 ++ wrolpi/status.py | 120 ++++++++++++-- wrolpi/switches.py | 2 +- wrolpi/test/test_common.py | 32 ++-- wrolpi/test/test_status.py | 23 +++ 14 files changed, 339 insertions(+), 282 deletions(-) diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 2c69cca3..cf861306 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -20,15 +20,7 @@ import { } from 'semantic-ui-react'; import {Link, NavLink, useNavigate, useSearchParams} from "react-router-dom"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; -import { - useCPUTemperature, - useHotspot, - useLoad, - useSearchDirectories, - useSearchOrder, - useThrottle, - useWROLMode -} from "../hooks/customHooks"; +import {useHotspot, useSearchDirectories, useSearchOrder, useThrottle, useWROLMode} from "../hooks/customHooks"; import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts"; import { Accordion, @@ -1124,38 +1116,6 @@ export function HotspotStatusIcon() { } -export function CPUTemperatureIcon({size = 'large', fallback = null}) { - // Returns an Icon only if temperature is too high. - const {temperature, highTemperature, criticalTemperature} = useCPUTemperature(); - - if (temperature === null || temperature < highTemperature) { - // Temperature is not a problem. - return fallback; - } - - let icon = temperature >= criticalTemperature ? - - : ; - icon = {icon}; - - return -} - -export function SystemLoadIcon({size = 'large', fallback = null}) { - const {minute_1, mediumLoad, highLoad} = useLoad(); - - if (!mediumLoad && !highLoad) { - return fallback; - } - - // Only return the icon if the load is not low. - const color = highLoad ? 'red' : 'yellow'; - let icon = - - ; - return -} - export function useTitle(title) { const documentDefined = typeof document !== 'undefined'; const originalTitle = React.useRef(documentDefined ? document.title : null); @@ -1950,3 +1910,11 @@ export function getDistinctColor(hexColors) { // If we've tried maxAttempts times, return the last generated color regardless return hslToHex(newColor.h, newColor.s, newColor.l); } + +export const isReactChildNull = (children) => { + // Returns `true` if a React element returns `null` as it's child. + const nonNullChildren = React.Children.map(children, + (child) => React.isValidElement(child) && child.type(child.props) + ); + return nonNullChildren.length === 0; +} diff --git a/app/src/components/Map.js b/app/src/components/Map.js index 3e1732e2..b40b1d20 100644 --- a/app/src/components/Map.js +++ b/app/src/components/Map.js @@ -99,7 +99,7 @@ const ViewerMessage = () => { function SlowImportMessage() { const {status} = useContext(StatusContext); - if (status && status['cpu_info'] && status['cpu_info']['temperature'] >= 80) { + if (status && status['cpu_stats'] && status['cpu_stats']['temperature'] >= 80) { return diff --git a/app/src/components/Nav.js b/app/src/components/Nav.js index 4d41942d..17d8ef9b 100644 --- a/app/src/components/Nav.js +++ b/app/src/components/Nav.js @@ -2,11 +2,11 @@ import React from "react"; import {Link, NavLink} from "react-router-dom"; import {Dropdown, Icon as SIcon, Menu} from "semantic-ui-react"; import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts"; -import {CPUTemperatureIcon, DarkModeToggle, HotspotStatusIcon, SystemLoadIcon, useLocalStorage} from "./Common"; +import {DarkModeToggle, HotspotStatusIcon, isReactChildNull, useLocalStorage} from "./Common"; import {ShareButton} from "./Share"; -import {useWROLMode} from "../hooks/customHooks"; +import {useCPUTemperature, useIOStats, useLoad, useMemoryStats, usePowerStats, useWROLMode} from "../hooks/customHooks"; import {SearchIconButton} from "./Search"; -import {Icon} from "./Theme"; +import {Icon, Popup} from "./Theme"; import {HELP_VIEWER_URI, NAME} from "./Vars"; import _ from "lodash"; @@ -59,15 +59,11 @@ function MenuLink({link}) { } } -function NavIconWrapper(props) { - if (_.isEmpty(props.children)) { - return <> - } else { - if (props.name === 'warning') { - // console.log(props.name, props.children); - } - return
{props.children}
- } +function NavIconWrapper({children}) { + // Do not add margins around empty react element to avoid filling up navbar. + const emptyChild = !children || isReactChildNull(children); + const style = emptyChild ? {} : {marginTop: '0.8em', marginLeft: '1.5em'}; + return
{children}
} function useNavColorSetting() { @@ -97,8 +93,74 @@ export function NavBar() { {topNavText} ; - // Display the temperature icon first because it can cause the system to throttle. - const warningIcon = }/>; + // Red/Yellow colors will blend in to some navbar colors, swap the colors for high contrast. + const conflictingColors = ['red', 'orange', 'yellow', 'olive', 'pink']; + const lowWarningColor = conflictingColors.includes(navColor) ? 'white' : 'yellow'; + const highWarningColor = conflictingColors.includes(navColor) ? 'black' : 'red'; + + // Generic system load, the least important warning. + const {minute_1, mediumLoad, highLoad} = useLoad(); + let systemLoadIcon; + if (mediumLoad || highLoad) { + // System load is high, display a warning icon. + const color = highLoad ? highWarningColor : lowWarningColor; + const icon = + + ; + systemLoadIcon = + } + + // RAM consumption. + const {percent: memoryPercent} = useMemoryStats(); + let memoryIcon; + if (memoryPercent > 80) { + const color = memoryPercent > 90 ? highWarningColor : lowWarningColor; + const icon = + + ; + memoryIcon = + } + + // Any disk is busy and processes are waiting. + const {percentIOWait} = useIOStats(); + let diskWaitIcon; + if (percentIOWait >= 50) { + // Processes are waiting on disk, display a warning icon. + const color = percentIOWait > 75 ? highWarningColor : lowWarningColor; + const icon = + + ; + diskWaitIcon = + } + + // CPU temperature. + const {temperature, highTemperature, criticalTemperature} = useCPUTemperature(); + let temperatureIcon; + if (temperature && temperature >= highTemperature) { + // CPU temperature is high, display a warning icon. + const color = temperature >= criticalTemperature ? highWarningColor : lowWarningColor; + const name = temperature >= criticalTemperature ? 'thermometer' : 'thermometer half'; + const icon = + const link = {icon}; + temperatureIcon = ; + } + + // Power issues, this is always displayed if detected. + const {underVoltage, overCurrent} = usePowerStats(); + let powerIcon; + if (underVoltage || overCurrent) { + const name = underVoltage ? 'power cord' : 'lightning'; + const icon = ; + const message = underVoltage + ? 'Under-voltage detected! Your power supply is insufficient!' + : 'Over-current detected! Your peripherals are using too much power!'; + powerIcon = ; + } + + // Display the temperature icon first because it can cause the system to throttle. The rest are in order of effects + // that will slow down the system and the user should address. Generic load is last because it is probably not an + // issue. + const warningIcon = temperatureIcon || diskWaitIcon || memoryIcon || systemLoadIcon; let processingLink; if (status && status.flags) { @@ -115,11 +177,12 @@ export function NavBar() { ; const icons = - {processingIcon} - {warningIcon} - - - + {processingIcon} + {powerIcon} + {warningIcon} + + + ; return <> diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index e2bcc34b..86e5aeb5 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -877,39 +877,29 @@ export const StatusProvider = (props) => { export const useStatusFlag = (flag) => { const {status} = useContext(StatusContext); - return status && status['flags'] && status['flags'][flag]; + return status?.flags?.[flag] || false; } export const useCPUTemperature = () => { const {status} = React.useContext(StatusContext); - let temperature = 0; - let highTemperature = 75; - let criticalTemperature = 85; - - if (status && status['cpu_info']) { - temperature = status['cpu_info']['temperature']; - highTemperature = status['cpu_info']['high_temperature']; - criticalTemperature = status['cpu_info']['critical_temperature']; - } - + const temperature = status?.cpu_stats?.temperature || 0; + const highTemperature = status?.cpu_stats?.high_temperature || 75; + const criticalTemperature = status?.cpu_stats?.critical_temperature || 85; return {temperature, highTemperature, criticalTemperature} } export const useLoad = () => { const {status} = React.useContext(StatusContext); - let minute_1; - let minute_5; - let minute_15; + let minute_1 = status?.load_stats?.minute_1; + let minute_5 = status?.load_stats?.minute_5; + let minute_15 = status?.load_stats?.minute_15; let mediumLoad = false; let highLoad = false; let cores; - if (status && status['load']) { - minute_1 = status['load']['minute_1']; - minute_5 = status['load']['minute_5']; - minute_15 = status['load']['minute_15']; - - cores = status['cpu_info']['cores']; + if (status && status.cpu_stats) { + // Medium load when 1/2 cores are busy, High load when 3/4+ cores are busy. + cores = status.cpu_stats.cores; const quarter = cores / 4; if (cores && minute_1 >= (quarter * 3)) { highLoad = true; @@ -921,6 +911,37 @@ export const useLoad = () => { return {minute_1, minute_5, minute_15, mediumLoad, highLoad, cores}; } +export const useIOStats = () => { + const {status} = React.useContext(StatusContext); + const percentIdle = status?.iostat_stats?.percent_idle; + const percentIOWait = status?.iostat_stats?.percent_iowait; + const percentNice = status?.iostat_stats?.percent_nice; + const percentSteal = status?.iostat_stats?.percent_steal; + const percentSystem = status?.iostat_stats?.percent_system; + const percentUser = status?.iostat_stats?.percent_user; + + return {percentIdle, percentIOWait, percentNice, percentSteal, percentSystem, percentUser}; +} + +export const useMemoryStats = () => { + const {status} = React.useContext(StatusContext); + const total = status?.memory_stats?.total; + const used = status?.memory_stats?.used; + const free = status?.memory_stats?.free; + const cached = status?.memory_stats?.cached; + let percent; + if (total && used) { + percent = Math.round((used / total) * 100); + } + return {total, used, free, cached, percent} +} + +export const usePowerStats = () => { + const {status} = React.useContext(StatusContext); + const underVoltage = status?.power_stats?.under_voltage; + const overCurrent = status?.power_stats?.over_current; + return {underVoltage, overCurrent}; +} export const useVideoStatistics = () => { const [statistics, setStatistics] = useState(null); diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot index 94cc05db..40d6555c 100644 --- a/debian-live-config/config/package-lists/wrolpi.list.chroot +++ b/debian-live-config/config/package-lists/wrolpi.list.chroot @@ -91,6 +91,7 @@ rsync smartmontools software-properties-common sudo +sysstat tar tcpdump tmux diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index f334769d..960a9cb3 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /opt/wrolpi # Update dependencies for the services install. RUN apt update # ffmpeg for Videos -RUN apt-get install -y ffmpeg catdoc +RUN apt-get install -y ffmpeg catdoc sysstat RUN ffmpeg -version # yt-dlp needs cache directory RUN mkdir /.cache && chown -R 1000:1000 /.cache diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages index ce36a6f9..da0cdf91 100644 --- a/pi-gen/stage2/04-wrolpi/01-packages +++ b/pi-gen/stage2/04-wrolpi/01-packages @@ -88,6 +88,7 @@ renderd rsync smartmontools software-properties-common +sysstat tar tcpdump tmux diff --git a/wrolpi/common.py b/wrolpi/common.py index f7fb91af..ae843988 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -21,7 +21,6 @@ from dataclasses import dataclass, asdict, field, fields from datetime import datetime, date from decimal import Decimal from functools import wraps -from http import HTTPStatus from itertools import islice, filterfalse, tee from multiprocessing.managers import DictProxy from pathlib import Path @@ -40,8 +39,8 @@ from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session -from wrolpi.dates import now, from_timestamp, seconds_to_timestamp -from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError, InvalidConfig, \ +from wrolpi.dates import now, from_timestamp +from wrolpi.errors import WROLModeEnabled, NativeOnly, LogLevelError, InvalidConfig, \ ValidationError from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS @@ -240,6 +239,7 @@ __all__ = [ 'apply_modelers', 'apply_refresh_cleanup', 'background_task', + 'can_connect_to_server', 'cancel_background_tasks', 'cancel_refresh_tasks', 'cancelable_wrapper', @@ -249,10 +249,10 @@ __all__ = [ 'chunks', 'chunks_by_stem', 'compile_tsvector', + 'create_empty_config_files', 'cum_timer', 'date_range', 'disable_wrol_mode', - 'download_file', 'enable_wrol_mode', 'escape_file_name', 'extract_domain', @@ -1414,48 +1414,6 @@ async def aiohttp_head(url: str, timeout: int = None, headers: dict = None) -> C yield response -async def speed_test(url: str, timeout: int | None = 10) -> int: - """Request the last megabyte of the provided url, return the content length divided by the time elapsed (speed).""" - async with aiohttp_head(url) as response: - content_length = response.headers['Content-Length'] - start_bytes = int(content_length) - 10485760 - range_ = f'bytes={start_bytes}-{content_length}' - start_time = datetime.now() - async with aiohttp_get(url, headers={'Range': range_}, timeout=timeout) as response: - content = await response.content.read() - elapsed = int((datetime.now() - start_time).total_seconds()) - return len(content) // elapsed - - -async def get_fastest_mirror(urls: List[str]) -> str: - """Perform a speed test on each URL, return the fastest.""" - fastest_url = urls[0] - fastest_speed = 0 - for url in urls: - try: - # Timeout after 5 minutes. Mirror is less than 27 kbps. - speed = await speed_test(url, timeout=60 * 5) - download_logger.debug(f'Speed test of {url} was {speed} bps') - except Exception as e: - download_logger.error(f'Speedtest of {url} failed', exc_info=e) - continue - - if speed > fastest_speed: - fastest_url = url - - return fastest_url - - -@dataclass -class DownloadFileInfoLink: - """Represents an HTTP "Link" Header.""" - url: str - rel: str - type: str - priority: int - geo: str - - @dataclass class DownloadFileInfo: """Information about a file that can be downloaded.""" @@ -1465,7 +1423,6 @@ class DownloadFileInfo: accept_ranges: str = None status: int = None location: str = None - links: List[DownloadFileInfoLink] = None FILENAME_MATCHER = re.compile(r'.*filename="(.*)"') @@ -1475,28 +1432,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo: """Gets information (name, size, etc.) about a downloadable file at the provided URL.""" async with aiohttp_head(url, timeout) as response: download_logger.debug(f'{response.headers=}') - try: - links = response.headers.getall('Link') - except KeyError: - links = None - - new_links = list() - if links: - # Convert "Link" header strings to DownloadFileInfoLink. - for idx, link in enumerate(links): - url, *props = link.split(';') - url = url[1:-1] - properties = dict() - for prop in props: - name, value = prop.strip().split('=') - properties[name] = value - new_links.append(DownloadFileInfoLink( - url, - properties.get('rel').strip() if 'rel' in properties else None, - properties.get('type').strip() if 'type' in properties else None, - int(properties.get('pri').strip()) if 'pri' in properties else None, - properties.get('geo').strip() if 'geo' in properties else None, - )) info = DownloadFileInfo( type=response.headers.get('Content-Type'), @@ -1504,7 +1439,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo: accept_ranges=response.headers.get('Accept-Ranges'), status=response.status, location=response.headers.get('Location'), - links=new_links, ) disposition = response.headers.get('Content-Disposition') @@ -1526,80 +1460,6 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo: download_logger = logger.getChild('download') -async def download_file(url: str, output_path: pathlib.Path = None, info: DownloadFileInfo = None, - timeout: int = 7 * 24 * 60 * 60): - """Uses aiohttp to download an HTTP file. Performs a speed test when mirrors are found, downloads from the fastest - mirror. - - Attempts to resume the file if `output_path` already exists. - - @warning: Timeout default is a week because of large downloads. - """ - info = info or await get_download_info(url, timeout) - - if output_path.is_file() and info.size == output_path.stat().st_size: - download_logger.warning(f'Already downloaded {repr(str(url))} to {repr(str(output_path))}') - return - - if info.links and (mirror_urls := [i.url for i in info.links if i.rel == 'duplicate']): - # Mirrors are available, find the fastest. - download_logger.info(f'Performing download speed test on mirrors: {mirror_urls}') - url = await get_fastest_mirror(mirror_urls) - info = await get_download_info(url, timeout) - - download_logger.debug(f'Final DownloadInfo fetched {info}') - total_size = info.size - - download_logger.info(f'Starting download of {url} with {total_size} total bytes') - if info.accept_ranges == 'bytes' or not output_path.is_file(): - with output_path.open('ab') as fh: - headers = DEFAULT_HTTP_HEADERS.copy() - # Check the position of append, if it is 0 then we do not need to resume. - position = fh.tell() - if position: - headers['Range'] = f'bytes={position}-' - - async with aiohttp_session(timeout=timeout) as session: - async with session.get(url, headers=headers) as response: - response: ClientResponse - if response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE: - download_logger.warning(f'Server responded with 416, file is probably already downloaded') - return - - if position and response.status != HTTPStatus.PARTIAL_CONTENT: - raise UnrecoverableDownloadError( - f'Tried to resume {repr(str(url))} but got status {response.status}') - - # May or may not be using Range. Append each chunk to the output file. - last_report = datetime.now() - bytes_received = 0 - async for data in response.content.iter_any(): - fh.write(data) - - bytes_received += len(data) - if (elapsed := (datetime.now() - last_report).total_seconds()) > 10: - # Report download speed every 10 seconds. - bytes_per_second = int(bytes_received // elapsed) - download_logger.debug(f'{bytes_received=} {elapsed=} {bytes_per_second=}') - size = fh.tell() - bytes_remaining = total_size - size - seconds_remaining = bytes_remaining // bytes_per_second - percent = int((size / total_size) * 100) - download_logger.info( - f'Downloading {url} at' - f' rate={human_bandwidth(bytes_per_second)}' - f' estimate={seconds_to_timestamp(seconds_remaining)}') - download_logger.debug(f'Downloading {url} {total_size=} {size=} {percent=}') - last_report = datetime.now() - bytes_received = 0 - - # Sleep to catch cancel. - await asyncio.sleep(0) - elif output_path.is_file(): - # TODO support downloading files that cannot be resumed. - raise UnrecoverableDownloadError(f'Cannot resume download {url}') - - def human_bandwidth(bps: int) -> str: """Convert bits per second to a more readable format. diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 56f30931..d33d47bb 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -9,7 +9,6 @@ import logging import multiprocessing import pathlib import shutil -import socketserver import tempfile import threading import zipfile @@ -760,26 +759,51 @@ def make_multipart_form(): @pytest.fixture -def simple_web_server(): - """Start a simple HTTP server in a background thread and yield its URL.""" - # Define the handler to serve files from the current directory - handler = http.server.SimpleHTTPRequestHandler +def simple_file_server(test_directory): + # Use a random port for testing + server_address = ('', 0) # 0 means pick any available port - # Set up an HTTP server on a free port - with socketserver.TCPServer(("localhost", 0), handler) as httpd: - # Get the port where the server is running - port = httpd.server_address[1] - server_url = f"http://localhost:{port}/" + class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + # Pass the directory to the handler + self.directory = kwargs['directory'] + super().__init__(*args, **kwargs) - # Start the server in a background thread - server_thread = threading.Thread(target=httpd.serve_forever) - server_thread.daemon = True - server_thread.start() + # Create the server + server = http.server.HTTPServer( + server_address, + # Hack to pass `test_directory` as `self.directory` to `SimpleHTTPRequestHandler` + lambda *args, **kwargs: + CustomHTTPRequestHandler(*args, directory=str(test_directory), **kwargs)) - # Yield the URL of the running server - try: - yield httpd, server_url - finally: - # Stop the server after the test - httpd.shutdown() - server_thread.join() + # Start the server in a background thread + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True # So the server dies when the main thread dies + server_thread.start() + + try: + yield server + finally: + # Cleanup + server.shutdown() + server.server_close() + server_thread.join() + + +@pytest.fixture +def mock_downloader_download_file(): + """Mock the Downloader.download_file method.""" + contents = None + + async def _download_file(self, id_, url, destination): + name = url.split('/')[-1] + output_path = destination / name + output_path.write_bytes(contents or b'') + return output_path + + with mock.patch('wrolpi.downloader.Downloader.download_file', _download_file): + def set_contents(contents_): + nonlocal contents + contents = contents_ + + yield set_contents diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 05985d29..2c4650d3 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -175,3 +175,15 @@ class RefreshConflict(APIError): code = 'REFRESH_CONFLICT' message = 'Not possible during file refresh' status_code = HTTPStatus.CONFLICT + + +class IgnoredDirectoryError(APIError): + code = 'IGNORED_DIRECTORY_ERROR' + message = 'Directory is being ignored' + status_code = HTTPStatus.BAD_REQUEST + + +class DownloadError(APIError): + code = 'DOWNLOAD_ERROR' + message = 'Unable to complete download' + status_code = HTTPStatus.INTERNAL_SERVER_ERROR diff --git a/wrolpi/status.py b/wrolpi/status.py index b7f84a0d..e4ee7efb 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -31,6 +31,7 @@ UPTIME_BIN = which('uptime', '/usr/bin/uptime') warn_once = get_warn_once('Unable to use psutil', logger) warn_cpu_once = get_warn_once('Cannot read CPU stats', logger) warn_cpu_temperature_once = get_warn_once('Cannot read CPU Temperature stats', logger) +warn_iostat_once = get_warn_once('Cannot get iostat stats', logger) status_worker_warn_once = get_warn_once('Status worker encountered error', logger) @@ -62,6 +63,58 @@ async def get_load_stats() -> SystemLoad: return load +@dataclass +class IostatInfo: + percent_idle: float = None + percent_iowait: float = None + percent_nice: float = None + percent_steal: float = None + percent_system: float = None + percent_user: float = None + + def __json__(self) -> dict: + return dict( + percent_idle=self.percent_idle, + percent_iowait=self.percent_iowait, + percent_nice=self.percent_nice, + percent_steal=self.percent_steal, + percent_system=self.percent_system, + percent_user=self.percent_user, + ) + + +async def get_iostat_stats() -> IostatInfo: + logger.trace(f'get_iostat_stats called') + + try: + proc = await asyncio.create_subprocess_shell( + 'iostat', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if stdout and proc.returncode == 0: + iostat_stats = jc.parse('iostat', stdout.decode(), quiet=True) + if iostat_stats: + cpu_stats, *_ = iostat_stats + + logger.debug( + f'get_iostat_stats got percent_iowait={cpu_stats["percent_iowait"]} percent_idle={cpu_stats["percent_idle"]}') + # Return only the CPU stats. + return IostatInfo( + percent_idle=cpu_stats['percent_idle'], + percent_iowait=cpu_stats['percent_iowait'], + percent_nice=cpu_stats['percent_nice'], + percent_steal=cpu_stats['percent_steal'], + percent_system=cpu_stats['percent_system'], + percent_user=cpu_stats['percent_user'], + ) + except Exception as e: + warn_iostat_once(e) + + return IostatInfo() + + @dataclass class ProcessInfo: pid: int = None @@ -113,16 +166,9 @@ async def get_cpu_stats() -> CPUInfo: """Get core count, max freq, min freq, current freq, cpu temperature.""" percent = int(psutil.cpu_percent()) - # Get temperature from system file. - temperature = cur_frequency = max_frequency = min_frequency = None - if TEMPERATURE_PATH.is_file(): - try: - temperature = int(TEMPERATURE_PATH.read_text()) // 1000 - min_frequency = int(MIN_FREQUENCY_PATH.read_text()) - max_frequency = int(MAX_FREQUENCY_PATH.read_text()) - cur_frequency = int(CUR_FREQUENCY_PATH.read_text()) - except Exception as e: - warn_cpu_once(e) + min_frequency = int(MIN_FREQUENCY_PATH.read_text()) + max_frequency = int(MAX_FREQUENCY_PATH.read_text()) + cur_frequency = int(CUR_FREQUENCY_PATH.read_text()) # Get temperature using psutil. temperature = high_temperature = critical_temperature = None @@ -480,6 +526,41 @@ async def get_disk_counters(shared_status): return disk_bandwidth_stats +@dataclass +class PowerStats: + under_voltage: bool = False + over_current: bool = False + + def __json__(self) -> dict: + return dict( + under_voltage=self.under_voltage, + over_current=self.over_current, + ) + + +async def get_power_stats() -> PowerStats: + logger.trace(f'get_power_stats called') + + try: + proc = await asyncio.create_subprocess_shell( + 'dmesg', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if stdout and proc.returncode == 0: + under_voltage = b' Undervoltage detected' in stdout + over_current = b' over-current change ' in stdout + return PowerStats( + under_voltage=under_voltage, + over_current=over_current, + ) + except Exception as e: + warn_iostat_once(e) + + return PowerStats() + + @perpetual_signal(sleep=5) async def status_worker(count: int = None, sleep_time: int = 5): """A background process which will gather historical data about system statistics.""" @@ -492,13 +573,16 @@ async def status_worker(count: int = None, sleep_time: int = 5): load_stats = None try: # Update global `status` dict with stats that are gathered instantly. - cpu_stats, load_stats, drives_stats, memory_stats, processes_stats = await asyncio.gather( - get_cpu_stats(), - get_load_stats(), - get_drives_stats(), - get_memory_stats(), - get_processes_stats(), - ) + cpu_stats, load_stats, drives_stats, memory_stats, processes_stats, iostat_stats, power_stats = \ + await asyncio.gather( + get_cpu_stats(), + get_load_stats(), + get_drives_stats(), + get_memory_stats(), + get_processes_stats(), + get_iostat_stats(), + get_power_stats(), + ) shared_status.update({ 'cpu_stats': cpu_stats.__json__(), 'load_stats': load_stats.__json__(), @@ -506,6 +590,8 @@ async def status_worker(count: int = None, sleep_time: int = 5): 'processes_stats': [i.__json__() for i in processes_stats], 'memory_stats': memory_stats.__json__(), 'last_status': now().isoformat(), + 'iostat_stats': iostat_stats.__json__(), + 'power_stats': power_stats.__json__(), }) if 'disk_bandwidth_stats' not in shared_status: diff --git a/wrolpi/switches.py b/wrolpi/switches.py index e09a8c09..908ca4f6 100644 --- a/wrolpi/switches.py +++ b/wrolpi/switches.py @@ -146,4 +146,4 @@ async def await_switches(timeout: int = 10): # All switches handled. break else: - raise RuntimeError('Timed out waiting for switches') + raise RuntimeError('Timed out waiting for switches. Did you remember to use the `await_switches` fixture?') diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 4838d376..e386a8de 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -834,24 +834,22 @@ def test_config_backup(async_client, test_wrolpi_config, test_directory): @pytest.mark.asyncio @skip_circleci -async def test_aiohttp(simple_web_server): +async def test_aiohttp(simple_file_server): """Test that aiohttp convenience functions work.""" - httpd, server_url = simple_web_server + host, port = simple_file_server.server_address + url = f'http://{host}:{port}' # noqa - async with common.aiohttp_get(server_url) as response: + async with common.aiohttp_get(url) as response: assert response.content.read() assert response.status == 200 - async with common.aiohttp_head(server_url) as response: + async with common.aiohttp_head(url) as response: assert response.content.read() assert response.status == 200 -def test_can_connect_to_server(simple_web_server): - httpd, server_url = simple_web_server - _, host, port = server_url.split(':') - host = host[2:] - port = int(port[:-1]) +def test_can_connect_to_server(simple_file_server): + host, port = simple_file_server.server_address assert common.can_connect_to_server(host, port) @@ -1121,28 +1119,28 @@ def test_get_paths_in_parent_directory(test_directory, make_files_structure): paths = list(common.walk(test_directory)) - assert common.get_paths_in_parent_directory(paths, test_directory / 'foo') == [ + assert set(common.get_paths_in_parent_directory(paths, test_directory / 'foo')) == { test_directory / 'foo/bar.txt', test_directory / 'foo/bar', test_directory / 'foo/bar/baz.txt', - ] + } - assert common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar') == [ + assert set(common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar')) == { test_directory / 'foo/bar/baz.txt', - ] + } - assert common.get_paths_in_parent_directory(paths, test_directory) == [ + assert set(common.get_paths_in_parent_directory(paths, test_directory)) == { test_directory / 'qux.txt', test_directory / 'foo', test_directory / 'foo/bar.txt', test_directory / 'foo/bar', test_directory / 'foo/bar/baz.txt', - ] + } - assert common.get_paths_in_media_directory(paths) == [ + assert set(common.get_paths_in_media_directory(paths)) == { test_directory / 'qux.txt', test_directory / 'foo', test_directory / 'foo/bar.txt', test_directory / 'foo/bar', test_directory / 'foo/bar/baz.txt', - ] + } diff --git a/wrolpi/test/test_status.py b/wrolpi/test/test_status.py index 2b6636f2..3e853838 100644 --- a/wrolpi/test/test_status.py +++ b/wrolpi/test/test_status.py @@ -27,6 +27,21 @@ async def test_get_cpu_stats(async_client): assert isinstance(info.temperature, int) +@pytest.mark.asyncio +async def test_iostat_stats(async_client): + info = await status.get_iostat_stats() + assert isinstance(info, status.IostatInfo) + assert isinstance(info.percent_idle, float) + + +@pytest.mark.asyncio +async def test_power_stats(async_client): + info = await status.get_power_stats() + assert isinstance(info, status.PowerStats) + assert info.under_voltage is False + assert info.over_current is False + + @skip_circleci @pytest.mark.asyncio async def test_get_drives_stats(async_client): @@ -57,3 +72,11 @@ async def test_status_worker(async_client): assert 'bytes_recv' in list(api_app.shared_ctx.status['nic_bandwidth_stats'].values())[0] assert api_app.shared_ctx.status['disk_bandwidth_stats'] assert 'bytes_read' in list(api_app.shared_ctx.status['disk_bandwidth_stats'].values())[0] + + assert isinstance(api_app.shared_ctx.status.get('cpu_stats'), dict) + assert isinstance(api_app.shared_ctx.status.get('load_stats'), dict) + assert isinstance(api_app.shared_ctx.status.get('drives_stats'), list) + assert isinstance(api_app.shared_ctx.status.get('processes_stats'), list) + assert isinstance(api_app.shared_ctx.status.get('memory_stats'), dict) + assert isinstance(api_app.shared_ctx.status.get('iostat_stats'), dict) + assert isinstance(api_app.shared_ctx.status.get('power_stats'), dict)