diff --git a/app/src/api.js b/app/src/api.js index cd4e3a90..245cd733 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -175,6 +175,20 @@ export async function getDirectories(search_str) { return []; } +export async function getStatus() { + let response = await apiGet(`${API_URI}/status`); + if (response.status === 200) { + return await response.json(); + } else { + toast({ + type: 'error', + title: 'Unexpected server response', + description: 'Could not get server status', + time: 5000, + }); + } +} + export async function getSettings() { let response = await apiGet(`${API_URI}/settings`); if (response.status === 200) { diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 5d16a547..f69925ec 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -11,14 +11,14 @@ import { Icon, Loader, Modal, - Placeholder, + Placeholder, Progress, Statistic, Table } from "semantic-ui-react"; import { clearCompletedDownloads, clearFailedDownloads, getDownloads, - getSettings, + getSettings, getStatus, killDownload, postDownload, saveSettings @@ -27,7 +27,7 @@ import TimezoneSelect from 'react-timezone-select'; import { DisableDownloadsToggle, HelpPopup, - HotspotToggle, + HotspotToggle, humanFileSize, PageContainer, secondsToDate, secondsToFrequency, @@ -226,7 +226,7 @@ class Settings extends React.Component { label='Hotspot Device' disabled={disabled || hotspot_password === null} value={hotspot_device} - onChange={(e, d) => this.handleInputChange(e,'hotspot_device', d.value)} + onChange={(e, d) => this.handleInputChange(e, 'hotspot_device', d.value)} /> @@ -320,6 +320,124 @@ class WROLMode extends React.Component { } } +function LoadStatistic({label, value, cores}) { + let color; + const quarter = cores / 4; + if (cores && value >= (quarter * 3)) { + color = 'red'; + } else if (cores && value >= (quarter * 2)) { + color = 'yellow'; + } + return ; +} + +function DriveInfo({used, size, percent, mount}) { + let color; + if (percent >= 90) { + color = 'red'; + } else if (percent >= 80) { + color = 'orange'; + } + return +} + +function CPUTemperatureStatistic({value}) { + if (!value) { + return + } + let color; + if (value >= 75) { + color = 'red'; + } else if (value >= 50) { + color = 'yellow'; + } + return +} + +function CPUUsageProgress({value, label}) { + if (value === null) { + return + } + + let color = 'grey'; + if (value >= 90) { + color = 'red'; + } else if (value >= 70) { + color = 'orange'; + } else if (value >= 50) { + color = 'yellow'; + } else if (value >= 30) { + color = 'olive'; + } + return +} + +class Status extends React.Component { + + constructor(props) { + super(props); + this.state = { + status: null, + } + this.fetchStatus = this.fetchStatus.bind(this); + } + + async componentDidMount() { + await this.fetchStatus(); + this.intervalId = setInterval(this.fetchStatus, 1000 * 5); + } + + componentWillUnmount() { + clearInterval(this.intervalId); + } + + fetchStatus = async () => { + try { + let status = await getStatus(); + this.setState({status: status}); + } catch (e) { + console.error(e); + this.setState({status: null}); + } + } + + render() { + let body = Loading system status; + + if (this.state.status) { + const {cpu_info, load, drives} = this.state.status; + const {minute_1, minute_5, minute_15} = load; + const {percent, cores, temperature} = cpu_info; + + body = (<> + + + + + + + + + + +
Drive Usage
+ {drives.map(DriveInfo)} + ); + } + + return ( + + {body} + + ) + } +} + function ClearCompleteDownloads({callback}) { const [disabled, setDisabled] = React.useState(false); @@ -680,6 +798,7 @@ export default function Admin(props) { const links = [ {text: 'Downloads', to: '/admin', exact: true, key: 'admin'}, {text: 'Settings', to: '/admin/settings', key: 'settings'}, + {text: 'Status', to: '/admin/status', key: 'status'}, {text: 'WROL Mode', to: '/admin/wrol', key: 'wrol'}, ]; @@ -688,6 +807,7 @@ export default function Admin(props) { + ) diff --git a/requirements.txt b/requirements.txt index a691eb2b..109d6658 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,4 @@ SQLAlchemy==1.3.22 srt==3.5.1 websockets==10.1 webvtt-py==0.4.6 -yt-dlp==2022.6.22.1 +yt-dlp==2022.6.29 diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 2696cdc2..61950a76 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -198,14 +198,11 @@ class Downloader: start = now() proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, **kwargs) pid = proc.pid - logger.debug(f'{self} launched download process {pid=} for {url}') # Timeout can be any positive integer. Global download timeout takes precedence, unless it is 0. # A timeout of 0 means the download will never be killed. - if timeout is None: - timeout = self.timeout - if WROLPI_CONFIG.download_timeout and timeout and timeout > WROLPI_CONFIG.download_timeout: - timeout = WROLPI_CONFIG.download_timeout + timeout = WROLPI_CONFIG.download_timeout or timeout or self.timeout + logger.debug(f'{self} launched download process {pid=} {timeout=} for {url}') try: while True: @@ -412,7 +409,7 @@ class DownloadManager: result = DownloadResult(success=False, error=str(traceback.format_exc())) error_len = len(result.error) if result.error else 0 - logger.debug(f'Got {result.success=} from {downloader} with {error_len=}') + logger.debug(f'Got success={result.success} from {downloader} with {error_len=}') # Long-running, get the download again. with get_db_session(commit=True) as session_: diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 3094ab07..d481a1fc 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -17,7 +17,7 @@ from sanic.signals import Event from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi import admin +from wrolpi import admin, status from wrolpi.admin import HotspotStatus from wrolpi.common import set_sanic_url_parts, logger, get_config, wrol_mode_enabled, Base, get_media_directory, \ wrol_mode_check, native_only, set_wrol_mode @@ -314,6 +314,16 @@ async def throttle_off(_: Request): return response.empty(HTTPStatus.INTERNAL_SERVER_ERROR) +@root_api.get('/status') +@openapi.description('Get the status of CPU/load/etc.') +async def get_status(_: Request): + cpu_info = await status.get_cpu_info() + load = await status.get_load() + drives = await status.get_drives_info() + ret = dict(cpu_info=cpu_info, load=load, drives=drives) + return json_response(ret) + + class CustomJSONEncoder(json.JSONEncoder): def default(self, obj): diff --git a/wrolpi/status.py b/wrolpi/status.py new file mode 100644 index 00000000..68fcae1f --- /dev/null +++ b/wrolpi/status.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +import asyncio +import multiprocessing +import re +import subprocess +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from typing import List, Optional + +from wrolpi.common import logger + +from wrolpi.cmd import which + +logger = logger.getChild(__name__) + +UPTIME_BIN = which('uptime', '/usr/bin/uptime') + + +@dataclass +class SystemLoad: + minute_1: Decimal = 0 + minute_5: Decimal = 0 + minute_15: Decimal = 0 + + def __json__(self): + return dict( + minute_1=str(self.minute_1), + minute_5=str(self.minute_5), + minute_15=str(self.minute_15), + ) + + +LOAD_REGEX = re.compile(r'.+?load average: (.+?), (.+?), (.*)') + + +async def get_load() -> SystemLoad: + proc = await asyncio.subprocess.create_subprocess_shell( + str(UPTIME_BIN), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + + stdout = stdout.strip() + if proc.returncode != 0 or not stdout: + logger.warning(f'{UPTIME_BIN} exited with {proc.returncode} or was empty') + return SystemLoad() + + stdout = stdout.decode() + load = SystemLoad(*[Decimal(i) for i in LOAD_REGEX.search(stdout).groups()]) + return load + + +@dataclass +class CPUInfo: + cores: int = None + cur_frequency: int = None + max_frequency: int = None + min_frequency: int = None + percent: int = None + temperature: int = None + + def __json__(self): + return dict( + cores=self.cores, + cur_frequency=self.cur_frequency, + max_frequency=self.max_frequency, + min_frequency=self.min_frequency, + percent=self.percent, + temperature=self.temperature, + ) + + +MIN_FREQUENCY_PATH = Path('/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq') +MAX_FREQUENCY_PATH = Path('/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq') +CUR_FREQUENCY_PATH = Path('/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq') +TEMPERATURE_PATH = Path('/sys/class/thermal/thermal_zone0/temp') +TOP_REGEX = re.compile(r'^%Cpu\(s\):\s+(\d+\.\d+)', re.MULTILINE) + + +async def get_cpu_info() -> CPUInfo: + """Get core count, max freq, min freq, current freq, cpu temperature.""" + proc = await asyncio.create_subprocess_shell( + 'top -bn1', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + logger.warning(f'Unable to get CPU top with exit {proc.returncode}') + percent = None + else: + percent = int(Decimal(TOP_REGEX.search(stdout.decode()).groups()[0])) + + if not TEMPERATURE_PATH.is_file(): + logger.warning(f'CPU temperature file does not exist!') + return CPUInfo(percent=percent) + + try: + temperature = int(TEMPERATURE_PATH.read_text()) // 1000 + min_frequency = int(MIN_FREQUENCY_PATH.read_text()) + max_frequency = int(MAX_FREQUENCY_PATH.read_text()) + cur_frequency = int(CUR_FREQUENCY_PATH.read_text()) + except Exception as e: + logger.warning(f'Unable to get CPU info', exc_info=e) + return CPUInfo(percent=percent) + + info = CPUInfo( + cores=multiprocessing.cpu_count(), + cur_frequency=cur_frequency - min_frequency, # Current frequency is between minimum and maximum. + max_frequency=max_frequency, + min_frequency=min_frequency, + percent=percent, + temperature=temperature, + ) + return info + + +@dataclass +class DriveInfo: + mount: str = None + percent: int = 0 + size: int = 0 + used: int = 0 + + def __json__(self): + return dict( + mount=self.mount, + percent=self.percent, + size=self.size, + used=self.used, + ) + + +DRIVE_REGEX = re.compile(r'^.+?' # filesystem + r'\s+(\d+)K' # size + r'\s+(\d+)K' # used + r'\s+(\d+)K' # available + r'\s+(\d+)%' # use % + r'\s+(.*)', # mount + re.MULTILINE) +IGNORED_DRIVES = ['/boot', '/etc'] + + +async def get_drives_info() -> List[DriveInfo]: + proc = await asyncio.subprocess.create_subprocess_shell( + 'df --type btrfs --type ext4 --type ext3 --type ext2 --type vfat -Bk', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + drives = [] + for size, used, _, percent, mount in DRIVE_REGEX.findall(stdout.decode()): + mount = mount.strip() + if any(mount.startswith(i) for i in IGNORED_DRIVES): + continue + + size = int(size) * 1024 + used = int(used) * 1024 + + drives.append(DriveInfo( + mount=mount, + percent=int(percent), + size=size, + used=used, + )) + + drives = sorted(drives, key=lambda i: i.mount) + return drives + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + print(loop.run_until_complete(get_cpu_info())) + print(loop.run_until_complete(get_load())) + print(loop.run_until_complete(get_drives_info())) diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index ce210bba..6e3423dd 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -390,12 +390,6 @@ def test_process_runner_timeout(test_directory): elapsed = datetime.now() - start assert 1 < elapsed.total_seconds() < 2 - # Zero timeout means no timeout. - start = datetime.now() - downloader.process_runner('https://example.com', ('sleep', '8'), test_directory, timeout=0) - elapsed = datetime.now() - start - assert 7 < elapsed.total_seconds() < 10 - # This Downloader's timeout is ignored in favor of the global timeout. from wrolpi.common import WROLPI_CONFIG WROLPI_CONFIG.download_timeout = 3 diff --git a/wrolpi/version.py b/wrolpi/version.py index 89d8f370..dabeb96b 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.5.2-beta' +__version__ = '0.5.3-beta' import subprocess