From a1a444934e27b1bb44a8d703a2c9113d07f7b54e Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 29 Jun 2022 20:07:40 -0600 Subject: [PATCH] Updating timeout to match new timeout logic. Logging timeout. Simplifying timeout logic. Updating yt-dlp. Bumping version for status page. Ignoring /etc for docker. Using top instead of custom script. Warning when temperature file does not exist. Handling when CPU files are not available. Using swapping drive availablility for size. Reworking page format. Creating wrolpi status page. --- app/src/api.js | 14 +++ app/src/components/Admin.js | 128 +++++++++++++++++++++++- requirements.txt | 2 +- wrolpi/downloader.py | 9 +- wrolpi/root_api.py | 12 ++- wrolpi/status.py | 177 +++++++++++++++++++++++++++++++++ wrolpi/test/test_downloader.py | 6 -- wrolpi/version.py | 2 +- 8 files changed, 331 insertions(+), 19 deletions(-) create mode 100644 wrolpi/status.py diff --git a/app/src/api.js b/app/src/api.js index cd4e3a90..245cd733 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -175,6 +175,20 @@ export async function getDirectories(search_str) { return []; } +export async function getStatus() { + let response = await apiGet(`${API_URI}/status`); + if (response.status === 200) { + return await response.json(); + } else { + toast({ + type: 'error', + title: 'Unexpected server response', + description: 'Could not get server status', + time: 5000, + }); + } +} + export async function getSettings() { let response = await apiGet(`${API_URI}/settings`); if (response.status === 200) { diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 5d16a547..f69925ec 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -11,14 +11,14 @@ import { Icon, Loader, Modal, - Placeholder, + Placeholder, Progress, Statistic, Table } from "semantic-ui-react"; import { clearCompletedDownloads, clearFailedDownloads, getDownloads, - getSettings, + getSettings, getStatus, killDownload, postDownload, saveSettings @@ -27,7 +27,7 @@ import TimezoneSelect from 'react-timezone-select'; import { DisableDownloadsToggle, HelpPopup, - HotspotToggle, + HotspotToggle, humanFileSize, PageContainer, secondsToDate, secondsToFrequency, @@ -226,7 +226,7 @@ class Settings extends React.Component { label='Hotspot Device' disabled={disabled || hotspot_password === null} value={hotspot_device} - onChange={(e, d) => this.handleInputChange(e,'hotspot_device', d.value)} + onChange={(e, d) => this.handleInputChange(e, 'hotspot_device', d.value)} /> @@ -320,6 +320,124 @@ class WROLMode extends React.Component { } } +function LoadStatistic({label, value, cores}) { + let color; + const quarter = cores / 4; + if (cores && value >= (quarter * 3)) { + color = 'red'; + } else if (cores && value >= (quarter * 2)) { + color = 'yellow'; + } + return ; +} + +function DriveInfo({used, size, percent, mount}) { + let color; + if (percent >= 90) { + color = 'red'; + } else if (percent >= 80) { + color = 'orange'; + } + return +} + +function CPUTemperatureStatistic({value}) { + if (!value) { + return + } + let color; + if (value >= 75) { + color = 'red'; + } else if (value >= 50) { + color = 'yellow'; + } + return +} + +function CPUUsageProgress({value, label}) { + if (value === null) { + return + } + + let color = 'grey'; + if (value >= 90) { + color = 'red'; + } else if (value >= 70) { + color = 'orange'; + } else if (value >= 50) { + color = 'yellow'; + } else if (value >= 30) { + color = 'olive'; + } + return +} + +class Status extends React.Component { + + constructor(props) { + super(props); + this.state = { + status: null, + } + this.fetchStatus = this.fetchStatus.bind(this); + } + + async componentDidMount() { + await this.fetchStatus(); + this.intervalId = setInterval(this.fetchStatus, 1000 * 5); + } + + componentWillUnmount() { + clearInterval(this.intervalId); + } + + fetchStatus = async () => { + try { + let status = await getStatus(); + this.setState({status: status}); + } catch (e) { + console.error(e); + this.setState({status: null}); + } + } + + render() { + let body = Loading system status; + + if (this.state.status) { + const {cpu_info, load, drives} = this.state.status; + const {minute_1, minute_5, minute_15} = load; + const {percent, cores, temperature} = cpu_info; + + body = (<> + + + + + + + + + + +
Drive Usage
+ {drives.map(DriveInfo)} + ); + } + + return ( + + {body} + + ) + } +} + function ClearCompleteDownloads({callback}) { const [disabled, setDisabled] = React.useState(false); @@ -680,6 +798,7 @@ export default function Admin(props) { const links = [ {text: 'Downloads', to: '/admin', exact: true, key: 'admin'}, {text: 'Settings', to: '/admin/settings', key: 'settings'}, + {text: 'Status', to: '/admin/status', key: 'status'}, {text: 'WROL Mode', to: '/admin/wrol', key: 'wrol'}, ]; @@ -688,6 +807,7 @@ export default function Admin(props) { + ) diff --git a/requirements.txt b/requirements.txt index a691eb2b..109d6658 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,4 @@ SQLAlchemy==1.3.22 srt==3.5.1 websockets==10.1 webvtt-py==0.4.6 -yt-dlp==2022.6.22.1 +yt-dlp==2022.6.29 diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 2696cdc2..61950a76 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -198,14 +198,11 @@ class Downloader: start = now() proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, **kwargs) pid = proc.pid - logger.debug(f'{self} launched download process {pid=} for {url}') # Timeout can be any positive integer. Global download timeout takes precedence, unless it is 0. # A timeout of 0 means the download will never be killed. - if timeout is None: - timeout = self.timeout - if WROLPI_CONFIG.download_timeout and timeout and timeout > WROLPI_CONFIG.download_timeout: - timeout = WROLPI_CONFIG.download_timeout + timeout = WROLPI_CONFIG.download_timeout or timeout or self.timeout + logger.debug(f'{self} launched download process {pid=} {timeout=} for {url}') try: while True: @@ -412,7 +409,7 @@ class DownloadManager: result = DownloadResult(success=False, error=str(traceback.format_exc())) error_len = len(result.error) if result.error else 0 - logger.debug(f'Got {result.success=} from {downloader} with {error_len=}') + logger.debug(f'Got success={result.success} from {downloader} with {error_len=}') # Long-running, get the download again. with get_db_session(commit=True) as session_: diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 3094ab07..d481a1fc 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -17,7 +17,7 @@ from sanic.signals import Event from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi import admin +from wrolpi import admin, status from wrolpi.admin import HotspotStatus from wrolpi.common import set_sanic_url_parts, logger, get_config, wrol_mode_enabled, Base, get_media_directory, \ wrol_mode_check, native_only, set_wrol_mode @@ -314,6 +314,16 @@ async def throttle_off(_: Request): return response.empty(HTTPStatus.INTERNAL_SERVER_ERROR) +@root_api.get('/status') +@openapi.description('Get the status of CPU/load/etc.') +async def get_status(_: Request): + cpu_info = await status.get_cpu_info() + load = await status.get_load() + drives = await status.get_drives_info() + ret = dict(cpu_info=cpu_info, load=load, drives=drives) + return json_response(ret) + + class CustomJSONEncoder(json.JSONEncoder): def default(self, obj): diff --git a/wrolpi/status.py b/wrolpi/status.py new file mode 100644 index 00000000..68fcae1f --- /dev/null +++ b/wrolpi/status.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +import asyncio +import multiprocessing +import re +import subprocess +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from typing import List, Optional + +from wrolpi.common import logger + +from wrolpi.cmd import which + +logger = logger.getChild(__name__) + +UPTIME_BIN = which('uptime', '/usr/bin/uptime') + + +@dataclass +class SystemLoad: + minute_1: Decimal = 0 + minute_5: Decimal = 0 + minute_15: Decimal = 0 + + def __json__(self): + return dict( + minute_1=str(self.minute_1), + minute_5=str(self.minute_5), + minute_15=str(self.minute_15), + ) + + +LOAD_REGEX = re.compile(r'.+?load average: (.+?), (.+?), (.*)') + + +async def get_load() -> SystemLoad: + proc = await asyncio.subprocess.create_subprocess_shell( + str(UPTIME_BIN), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + + stdout = stdout.strip() + if proc.returncode != 0 or not stdout: + logger.warning(f'{UPTIME_BIN} exited with {proc.returncode} or was empty') + return SystemLoad() + + stdout = stdout.decode() + load = SystemLoad(*[Decimal(i) for i in LOAD_REGEX.search(stdout).groups()]) + return load + + +@dataclass +class CPUInfo: + cores: int = None + cur_frequency: int = None + max_frequency: int = None + min_frequency: int = None + percent: int = None + temperature: int = None + + def __json__(self): + return dict( + cores=self.cores, + cur_frequency=self.cur_frequency, + max_frequency=self.max_frequency, + min_frequency=self.min_frequency, + percent=self.percent, + temperature=self.temperature, + ) + + +MIN_FREQUENCY_PATH = Path('/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq') +MAX_FREQUENCY_PATH = Path('/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq') +CUR_FREQUENCY_PATH = Path('/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq') +TEMPERATURE_PATH = Path('/sys/class/thermal/thermal_zone0/temp') +TOP_REGEX = re.compile(r'^%Cpu\(s\):\s+(\d+\.\d+)', re.MULTILINE) + + +async def get_cpu_info() -> CPUInfo: + """Get core count, max freq, min freq, current freq, cpu temperature.""" + proc = await asyncio.create_subprocess_shell( + 'top -bn1', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + logger.warning(f'Unable to get CPU top with exit {proc.returncode}') + percent = None + else: + percent = int(Decimal(TOP_REGEX.search(stdout.decode()).groups()[0])) + + if not TEMPERATURE_PATH.is_file(): + logger.warning(f'CPU temperature file does not exist!') + return CPUInfo(percent=percent) + + try: + temperature = int(TEMPERATURE_PATH.read_text()) // 1000 + min_frequency = int(MIN_FREQUENCY_PATH.read_text()) + max_frequency = int(MAX_FREQUENCY_PATH.read_text()) + cur_frequency = int(CUR_FREQUENCY_PATH.read_text()) + except Exception as e: + logger.warning(f'Unable to get CPU info', exc_info=e) + return CPUInfo(percent=percent) + + info = CPUInfo( + cores=multiprocessing.cpu_count(), + cur_frequency=cur_frequency - min_frequency, # Current frequency is between minimum and maximum. + max_frequency=max_frequency, + min_frequency=min_frequency, + percent=percent, + temperature=temperature, + ) + return info + + +@dataclass +class DriveInfo: + mount: str = None + percent: int = 0 + size: int = 0 + used: int = 0 + + def __json__(self): + return dict( + mount=self.mount, + percent=self.percent, + size=self.size, + used=self.used, + ) + + +DRIVE_REGEX = re.compile(r'^.+?' # filesystem + r'\s+(\d+)K' # size + r'\s+(\d+)K' # used + r'\s+(\d+)K' # available + r'\s+(\d+)%' # use % + r'\s+(.*)', # mount + re.MULTILINE) +IGNORED_DRIVES = ['/boot', '/etc'] + + +async def get_drives_info() -> List[DriveInfo]: + proc = await asyncio.subprocess.create_subprocess_shell( + 'df --type btrfs --type ext4 --type ext3 --type ext2 --type vfat -Bk', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + drives = [] + for size, used, _, percent, mount in DRIVE_REGEX.findall(stdout.decode()): + mount = mount.strip() + if any(mount.startswith(i) for i in IGNORED_DRIVES): + continue + + size = int(size) * 1024 + used = int(used) * 1024 + + drives.append(DriveInfo( + mount=mount, + percent=int(percent), + size=size, + used=used, + )) + + drives = sorted(drives, key=lambda i: i.mount) + return drives + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + print(loop.run_until_complete(get_cpu_info())) + print(loop.run_until_complete(get_load())) + print(loop.run_until_complete(get_drives_info())) diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index ce210bba..6e3423dd 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -390,12 +390,6 @@ def test_process_runner_timeout(test_directory): elapsed = datetime.now() - start assert 1 < elapsed.total_seconds() < 2 - # Zero timeout means no timeout. - start = datetime.now() - downloader.process_runner('https://example.com', ('sleep', '8'), test_directory, timeout=0) - elapsed = datetime.now() - start - assert 7 < elapsed.total_seconds() < 10 - # This Downloader's timeout is ignored in favor of the global timeout. from wrolpi.common import WROLPI_CONFIG WROLPI_CONFIG.download_timeout = 3 diff --git a/wrolpi/version.py b/wrolpi/version.py index 89d8f370..dabeb96b 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.5.2-beta' +__version__ = '0.5.3-beta' import subprocess