diff --git a/app/src/Dashboard.js b/app/src/Dashboard.js
index bdb1e7ce..865ebf46 100644
--- a/app/src/Dashboard.js
+++ b/app/src/Dashboard.js
@@ -6,8 +6,8 @@ import {DownloadMenu} from "./components/Upload";
import {FilesSearchView} from "./components/Files";
import {Header, Segment, Statistic, StatisticGroup} from "./components/Theme";
import {Link} from "react-router-dom";
-import {CPUUsageProgress} from "./components/admin/Status";
-import {Divider} from "semantic-ui-react";
+import {BandwidthProgressCombined, CPUUsageProgress} from "./components/admin/Status";
+import {ProgressPlaceholder} from "./components/Placeholder";
export function Dashboard() {
useTitle('Dashboard');
@@ -69,15 +69,19 @@ function DashboardStatus() {
+
-
-
+
+ {status && status['bandwidth'] ?
+ status['bandwidth'].map(i => ) :
+ }
+
diff --git a/app/src/components/Common.js b/app/src/components/Common.js
index 988ac0b4..bb919e1a 100644
--- a/app/src/components/Common.js
+++ b/app/src/components/Common.js
@@ -355,8 +355,24 @@ export function humanFileSize(bytes, si = false, dp = 1) {
return bytes.toFixed(dp) + ' ' + units[u];
}
+export function humanBandwidth(bytes) {
+ // Convert bytes to Mbps and return a string.
+ bytes /= 125;
+ let size = 'Kbps';
+ if (bytes > 1000) {
+ bytes /= 125_000;
+ size = 'Mbps';
+ }
+ if (bytes > 1000) {
+ bytes /= 1000;
+ size = 'Gbps';
+ }
+ bytes = Math.round(bytes);
+ return `${bytes} ${size}`;
+}
+
export function humanNumber(num, dp = 1) {
- // Convert large numbers to a more human readable format.
+ // Convert large numbers to a more human-readable format.
// >> humanNumber(1000)
// 1.0k
// >> humanNumber(1500000)
diff --git a/app/src/components/Placeholder.js b/app/src/components/Placeholder.js
index b8da5ee6..84f16045 100644
--- a/app/src/components/Placeholder.js
+++ b/app/src/components/Placeholder.js
@@ -36,3 +36,10 @@ export function ChannelPlaceholder() {
)
}
+
+export function ProgressPlaceholder() {
+ return
+
+
+
+}
diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js
index 200356f4..f7481262 100644
--- a/app/src/components/Upload.js
+++ b/app/src/components/Upload.js
@@ -275,13 +275,13 @@ export function DownloadMenu({onOpen}) {
onOpen(name)
}
- let body = (
+ let body = (<>
);
+ );
function clearSelected() {
localOnOpen(null);
diff --git a/app/src/components/admin/Status.js b/app/src/components/admin/Status.js
index 70cee0e4..35dd285d 100644
--- a/app/src/components/admin/Status.js
+++ b/app/src/components/admin/Status.js
@@ -1,9 +1,10 @@
import {Header, Loader, Progress, Statistic, StatisticGroup} from "../Theme";
import React from "react";
-import {humanFileSize, LoadStatistic} from "../Common";
+import {humanBandwidth, humanFileSize, LoadStatistic} from "../Common";
import {getStatus} from "../../api";
-import {ThemeContext} from "../../contexts/contexts";
-import {Container} from "semantic-ui-react";
+import {Container, Divider} from "semantic-ui-react";
+import {ProgressPlaceholder} from "../Placeholder";
+import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
function DriveInfo({used, size, percent, mount}) {
let color;
@@ -20,17 +21,67 @@ function DriveInfo({used, size, percent, mount}) {
/>
}
-function CPUTemperatureStatistic({value}) {
+function CPUTemperatureStatistic({value, high_temperature, critical_temperature}) {
if (!value) {
return
}
- let color;
- if (value >= 75) {
- color = 'red';
- } else if (value >= 50) {
- color = 'yellow';
+ const props = {};
+ if ((critical_temperature && value >= critical_temperature) || (!critical_temperature && value >= 75)) {
+ props['color'] = 'red';
+ } else if ((high_temperature && value >= high_temperature) || (!high_temperature && value >= 60)) {
+ props['color'] = 'orange';
}
- return
+ return
+}
+
+export function BandwidthProgress({label = '', bytes, maxBytes, ...props}) {
+ // Gigabit by default.
+ maxBytes = maxBytes || 125_000_000;
+
+ label = `${label} (${humanBandwidth(bytes)})`;
+ const percent = (bytes / maxBytes);
+ if (percent > 70) {
+ props['color'] = 'yellow';
+ } else if (percent > 90) {
+ props['color'] = 'red';
+ }
+ return
+}
+
+export function BandwidthProgressGroup({bandwidth, ...props}) {
+ // NIC speed to bytes.
+ const maxBytes = bandwidth['speed'] * 125_000;
+
+ const recv = ;
+
+ const sent = ;
+
+ return
+
+
+ {recv}
+
+
+ {sent}
+
+
+
+}
+
+export function BandwidthProgressCombined({bandwidth , ...props}) {
+ const maxBytes = bandwidth['speed'] ? bandwidth['speed'] * 125_000 : 125_000_000;
+ const combined = bandwidth['bytes_recv'] + bandwidth['bytes_sent']
+ return
}
export function CPUUsageProgress({value, label}) {
@@ -82,33 +133,44 @@ export class Status extends React.Component {
render() {
if (this.state.status) {
- const {cpu_info, load, drives} = this.state.status;
+ const {cpu_info, load, drives, bandwidth} = this.state.status;
const {minute_1, minute_5, minute_15} = load;
- const {percent, cores, temperature} = cpu_info;
+ const {percent, cores, temperature, high_temperature, critical_temperature} = cpu_info;
- return
- {({i}) => <>
-
+ return <>
+
-
-
-
-
-
-
-
+
+
-
- {drives.map((drive) => )}
- }
-
+
+
+
+
+
+
+
+
+
+
+
+ {bandwidth ?
+ bandwidth.map(i => ) :
+ }
+
+
+
+
+ {drives.map((drive) => )}
+
}
- return
- {({i}) =>
- Loading system status;
-
- }
-
+ return
+ Loading system status
+
}
}
diff --git a/docker/archive/main.py b/docker/archive/main.py
index 2bbb37ba..73306cf4 100644
--- a/docker/archive/main.py
+++ b/docker/archive/main.py
@@ -23,6 +23,12 @@ formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(messag
ch.setFormatter(formatter)
logger.addHandler(ch)
+SINGLEFILE_PATH = pathlib.Path('/usr/src/app/node_modules/single-file-cli/single-file')
+if not SINGLEFILE_PATH.is_file():
+ SINGLEFILE_PATH = pathlib.Path('/usr/src/app/node_modules/single-file/cli/single-file')
+if not SINGLEFILE_PATH.is_file():
+ raise FileNotFoundError("Can't find single-file executable!")
+
# Increase response timeout, archiving can take several minutes.
RESPONSE_TIMEOUT = 10 * 60
config = {
@@ -57,13 +63,22 @@ async def call_single_file(url) -> bytes:
See https://github.com/gildas-lormeau/SingleFile
"""
logger.info(f'archiving {url}')
- cmd = ['/usr/src/app/node_modules/single-file-cli/single-file', url,
- '--browser-executable-path', '/usr/bin/chromium-browser', '--browser-args', '["--no-sandbox"]',
- '--dump-content']
+ cmd = f'{SINGLEFILE_PATH}' \
+ r' --browser-executable-path /usr/bin/chromium-browser' \
+ r' --browser-args [\"--no-sandbox\"]' \
+ r' --dump-content ' \
+ f' {url}'
logger.debug(f'archive cmd: {cmd}')
- output = subprocess.check_output(cmd)
+ proc = await asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = await proc.communicate()
+
+ if stderr:
+ for line in stderr.decode().splitlines():
+ logger.error(line)
+ if proc.returncode != 0 or not stdout:
+ raise ValueError(f'Failed to single-file {url}')
logger.debug(f'done archiving for {url}')
- return output
+ return stdout
async def extract_readability(path: str, url: str) -> dict:
@@ -133,8 +148,8 @@ async def post_archive(request: Request):
screenshot=screenshot,
)
return response.json(ret)
- except Exception:
- logger.fatal(f'Failed to archive {url}', exc_info=True)
+ except Exception as e:
+ logger.error(f'Failed to archive {url}', exc_info=e)
return response.json({'error': f'Failed to archive {url}'})
diff --git a/main.py b/main.py
index 5f67766e..6db45109 100755
--- a/main.py
+++ b/main.py
@@ -209,6 +209,13 @@ async def start_workers(app: Sanic, loop):
download_manager.start_workers()
+@after_startup
+@limit_concurrent(1)
+async def bandwidth_worker(app: Sanic, loop):
+ from wrolpi import status
+ await app.add_task(status.bandwidth_worker())
+
+
@root_api.api_app.signal(Event.SERVER_SHUTDOWN_BEFORE)
@limit_concurrent(1)
def handle_server_shutdown(*args, **kwargs):
diff --git a/requirements.txt b/requirements.txt
index f171ac25..5483c93c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,10 +8,12 @@ beautifulsoup4==4.10.0
cachetools==5.0.0
feedparser==6.0.9
mock==4.0.3
+psutil==5.9.2
psycopg2-binary==2.9.3
pytest-asyncio==0.17.2
pytest==6.2.5
python-dotenv==0.19.2
+python-magic~=0.4.27
pytz==2021.3
sanic-ext==22.1.2
sanic-testing==0.8.2
@@ -21,4 +23,3 @@ srt==3.5.1
websockets==10.1
webvtt-py==0.4.6
yt-dlp==2022.9.1
-python-magic~=0.4.27
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 8b189163..f224498a 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -312,11 +312,9 @@ async def throttle_off(_: Request):
@root_api.get('/status')
@openapi.description('Get the status of CPU/load/etc.')
async def get_status(_: Request):
- cpu_info = await status.get_cpu_info()
- load = await status.get_load()
- drives = await status.get_drives_info()
+ s = await status.get_status()
downloads = download_manager.get_summary()
- ret = dict(cpu_info=cpu_info, load=load, drives=drives, downloads=downloads)
+ ret = dict(downloads=downloads, **s.__dict__)
return json_response(ret)
diff --git a/wrolpi/status.py b/wrolpi/status.py
index 22536f31..0dc11d44 100755
--- a/wrolpi/status.py
+++ b/wrolpi/status.py
@@ -6,14 +6,31 @@ import subprocess
from dataclasses import dataclass
from decimal import Decimal
from pathlib import Path
-from typing import List
+from typing import List, Optional, Tuple
from wrolpi.cmd import which
from wrolpi.common import logger
+from wrolpi.dates import now
+
+try:
+ import psutil
+except ImportError:
+ logger.warning('Unable to import psutil!')
+ psutil = None
logger = logger.getChild(__name__)
UPTIME_BIN = which('uptime', '/usr/bin/uptime')
+PSUTIL_WARNED = multiprocessing.Event()
+
+
+def warn_once(exception: Exception):
+ """
+ Don't spam the logs with errors when status can't use psutil.
+ """
+ if not PSUTIL_WARNED.is_set():
+ logger.error(f'Unable to use psutil', exc_info=exception)
+ PSUTIL_WARNED.set()
@dataclass
@@ -33,7 +50,18 @@ class SystemLoad:
LOAD_REGEX = re.compile(r'.+?load average: (.+?), (.+?), (.*)')
+def get_load_psutil() -> SystemLoad:
+ load = SystemLoad(*list(map(Decimal, psutil.getloadavg())))
+ return load
+
+
async def get_load() -> SystemLoad:
+ try:
+ return get_load_psutil()
+ except Exception as e:
+ warn_once(e)
+
+ # Fallback to using `uptime` to fetch load information.
proc = await asyncio.subprocess.create_subprocess_shell(
str(UPTIME_BIN),
stdout=subprocess.PIPE,
@@ -58,7 +86,9 @@ class CPUInfo:
max_frequency: int = None
min_frequency: int = None
percent: int = None
- temperature: int = None
+ temperature: Optional[int] = None
+ high_temperature: Optional[int] = None
+ critical_temperature: Optional[int] = None
def __json__(self):
return dict(
@@ -68,6 +98,8 @@ class CPUInfo:
min_frequency=self.min_frequency,
percent=self.percent,
temperature=self.temperature,
+ high_temperature=self.high_temperature,
+ critical_temperature=self.critical_temperature,
)
@@ -78,8 +110,38 @@ TEMPERATURE_PATH = Path('/sys/class/thermal/thermal_zone0/temp')
TOP_REGEX = re.compile(r'^%Cpu\(s\):\s+(\d+\.\d+)', re.MULTILINE)
+def get_cpu_info_psutil() -> CPUInfo:
+ cpu_freq = psutil.cpu_freq()
+ percent = psutil.cpu_percent(interval=1)
+
+ # Prefer "coretemp", fallback to the first temperature.
+ temp = psutil.sensors_temperatures()
+ name = 'coretemp' if 'coretemp' in temp else list(temp.keys())[0]
+ temperature = int(temp.get(name)[0].current)
+ high_temperature = int(temp.get(name)[0].high)
+ critical_temperature = int(temp.get(name)[0].critical)
+
+ info = CPUInfo(
+ cores=psutil.cpu_count(logical=True),
+ cur_frequency=int(cpu_freq.current),
+ min_frequency=int(cpu_freq.min),
+ max_frequency=int(cpu_freq.max),
+ percent=int(percent),
+ temperature=temperature,
+ high_temperature=high_temperature,
+ critical_temperature=critical_temperature,
+ )
+ return info
+
+
async def get_cpu_info() -> CPUInfo:
"""Get core count, max freq, min freq, current freq, cpu temperature."""
+ try:
+ return get_cpu_info_psutil()
+ except Exception as e:
+ warn_once(e)
+
+ # Fallback to using `top` to fetch CPU information.
proc = await asyncio.create_subprocess_shell(
'top -bn1',
stdout=subprocess.PIPE,
@@ -140,9 +202,39 @@ DRIVE_REGEX = re.compile(r'^.+?' # filesystem
r'\s+(.*)', # mount
re.MULTILINE)
IGNORED_DRIVES = ['/boot', '/etc']
+VALID_FORMATS = {'btrfs', 'ext4', 'ext3', 'ext2', 'vfat'}
+
+
+def get_drives_info_psutil() -> List[DriveInfo]:
+ info = {}
+
+ disks = psutil.disk_partitions()
+ for disk in disks:
+ if any(disk.mountpoint.startswith(i) for i in IGNORED_DRIVES):
+ continue
+ if disk.fstype not in VALID_FORMATS:
+ continue
+ if disk.device not in info:
+ # Only use the first use of the partition.
+ usage = psutil.disk_usage(disk.mountpoint)
+ info[disk.device] = DriveInfo(
+ mount=disk.mountpoint,
+ percent=int(usage.percent),
+ size=int(usage.total),
+ used=int(usage.used),
+ )
+
+ info = sorted(info.values(), key=lambda i: i.mount)
+ return info
async def get_drives_info() -> List[DriveInfo]:
+ try:
+ return get_drives_info_psutil()
+ except Exception as e:
+ warn_once(e)
+
+ # Fallback to using `df` to fetch disk information.
proc = await asyncio.subprocess.create_subprocess_shell(
'df --type btrfs --type ext4 --type ext3 --type ext2 --type vfat -Bk',
stdout=subprocess.PIPE,
@@ -169,8 +261,177 @@ async def get_drives_info() -> List[DriveInfo]:
return drives
+@dataclass
+class BandwidthInfo:
+ bytes_recv: int = None
+ bytes_sent: int = None
+ elapsed: int = None
+ name: str = None
+ speed: int = None
+
+ def __json__(self):
+ return dict(
+ bytes_recv=self.bytes_recv,
+ bytes_sent=self.bytes_sent,
+ elapsed=self.elapsed,
+ name=self.name,
+ speed=self.speed,
+ )
+
+
+IGNORED_NIC_NAMES = {
+ 'lo',
+ 'veth',
+ 'tun',
+ 'docker',
+ 'br-',
+}
+
+
+def get_nic_names() -> List[str]:
+ """Finds all non-virtual and non-docker network interface names."""
+ names = []
+ for name, nic in psutil.net_if_stats().items():
+ if any(name.startswith(i) for i in IGNORED_NIC_NAMES):
+ continue
+ names.append(name)
+ return names
+
+
+BANDWIDTH = multiprocessing.Manager().dict()
+
+
+async def get_bandwidth_info() -> List[BandwidthInfo]:
+ """
+ Get all bandwidth information for all NICs.
+ """
+ infos = []
+ try:
+ for name in sorted(BANDWIDTH.keys()):
+ nic = BANDWIDTH[name]
+ if 'bytes_recv_ps' not in nic:
+ # Not stats collected yet.
+ continue
+ infos.append(BandwidthInfo(
+ bytes_recv=nic['bytes_recv_ps'],
+ bytes_sent=nic['bytes_sent_ps'],
+ elapsed=nic['elapsed'],
+ name=name,
+ speed=nic['speed'],
+ ))
+ except Exception as e:
+ warn_once(e)
+
+ return infos
+
+
+def _get_nic_tick(name_):
+ """
+ Get the instant network statistics for the provided NIC.
+ """
+ try:
+ counter = psutil.net_io_counters(pernic=True, nowrap=True)[name_]
+ stats = psutil.net_if_stats()[name_]
+ return now().timestamp(), counter.bytes_recv, counter.bytes_sent, int(stats.speed)
+ except Exception as e:
+ warn_once(e)
+ return 0, 0, 0, 0
+
+
+def _calculate_bytes_per_second(history: List[Tuple]) -> Tuple[int, int, int]:
+ """
+ Calculate the bytes-per-second between the oldest and newest tick.
+ """
+ (oldest_now, oldest_recv, oldest_sent, _), (newest_now, newest_recv, newest_sent, _) = \
+ history[0], history[-1]
+ elapsed = int(newest_now - oldest_now)
+ bytes_recv_ps = int((newest_recv - oldest_recv) // elapsed)
+ bytes_sent_ps = int((newest_sent - oldest_sent) // elapsed)
+ return bytes_recv_ps, bytes_sent_ps, elapsed
+
+
+async def bandwidth_worker(count: int = None):
+ """
+ A background process which will gather historical data about all NIC bandwidth statistics.
+ """
+ if not psutil:
+ return
+
+ nic_names = get_nic_names()
+
+ def append_all_stats():
+ for name_ in nic_names:
+ nic = BANDWIDTH.get(name_)
+ if not nic:
+ # Initialize history for this NIC.
+ BANDWIDTH.update({
+ name_: dict(historical=[_get_nic_tick(name_), ]),
+ })
+ else:
+ # Append to history for this NIC.
+ nic['historical'] = (nic['historical'] + [_get_nic_tick(name_), ])[-21:]
+ BANDWIDTH.update({name_: nic})
+
+ # Initialize the stats.
+ append_all_stats()
+
+ while count is None or count > 0:
+ await asyncio.sleep(1)
+ if count is not None:
+ count -= 1
+
+ append_all_stats()
+
+ # Calculate the difference between the first and last bandwidth ticks for all NICs.
+ for name, nic in BANDWIDTH.items():
+ historical = nic['historical']
+ bytes_recv_ps, bytes_sent_ps, elapsed = _calculate_bytes_per_second(historical)
+ BANDWIDTH.update({
+ name: dict(
+ historical=historical,
+ bytes_recv_ps=bytes_recv_ps,
+ bytes_sent_ps=bytes_sent_ps,
+ elapsed=elapsed,
+ speed=historical[-1][-1], # Use the most recent speed.
+ )
+ })
+
+
+@dataclass
+class Status:
+ cpu_info: CPUInfo
+ load: SystemLoad
+ drives: List[DriveInfo]
+ bandwidth: BandwidthInfo
+
+
+async def get_status() -> Status:
+ cpu_info, load, drives, bandwidth = await asyncio.gather(
+ get_cpu_info(),
+ get_load(),
+ get_drives_info(),
+ get_bandwidth_info(),
+ )
+ return Status(
+ cpu_info=cpu_info,
+ load=load,
+ drives=drives,
+ bandwidth=bandwidth,
+ )
+
+
if __name__ == '__main__':
+ from pprint import pprint
+
loop = asyncio.get_event_loop()
- print(loop.run_until_complete(get_cpu_info()))
- print(loop.run_until_complete(get_load()))
- print(loop.run_until_complete(get_drives_info()))
+
+ loop.run_until_complete(bandwidth_worker(2))
+
+ task = asyncio.gather(
+ get_cpu_info(),
+ get_load(),
+ get_drives_info(),
+ get_bandwidth_info(),
+ )
+ info_ = loop.run_until_complete(task)
+ pprint(info_)
diff --git a/wrolpi/test/test_status.py b/wrolpi/test/test_status.py
index d1abe10e..8e790a84 100644
--- a/wrolpi/test/test_status.py
+++ b/wrolpi/test/test_status.py
@@ -1,4 +1,5 @@
from decimal import Decimal
+from unittest import mock
import pytest
@@ -14,6 +15,14 @@ async def test_get_load():
assert load.minute_5 is not None and isinstance(load.minute_5, Decimal)
assert load.minute_15 is not None and isinstance(load.minute_15, Decimal)
+ with mock.patch('wrolpi.status.get_load_psutil') as mock_get_load_psutil:
+ # Fallback to subprocess when psutil is not available.
+ mock_get_load_psutil.side_effect = Exception('testing no psutil')
+ load = await status.get_load()
+ assert isinstance(load, status.SystemLoad)
+ assert load.minute_1 is not None
+ assert isinstance(load.minute_1, Decimal)
+
@pytest.mark.asyncio
async def test_get_cpu_info():
@@ -22,6 +31,13 @@ async def test_get_cpu_info():
assert isinstance(info, status.CPUInfo)
assert isinstance(info.percent, int)
+ with mock.patch('wrolpi.status.get_cpu_info_psutil') as mock_get_cpu_info_psutil:
+ # Fallback to subprocess when psutil is not available.
+ mock_get_cpu_info_psutil.side_effect = Exception('testing no psutil')
+ info = await status.get_cpu_info()
+ assert isinstance(info, status.CPUInfo)
+ assert isinstance(info.percent, int)
+
@pytest.mark.asyncio
async def test_get_drives_info():
@@ -30,3 +46,35 @@ async def test_get_drives_info():
assert isinstance(info, list)
assert len(info) >= 1
assert isinstance(info[0], status.DriveInfo)
+
+ with mock.patch('wrolpi.status.get_drives_info_psutil') as mock_get_drives_info_psutil:
+ # Fallback to subprocess when psutil is not available.
+ mock_get_drives_info_psutil.side_effect = Exception('testing no psutil')
+ info = await status.get_drives_info()
+ assert isinstance(info, list)
+ assert len(info) >= 1
+ assert isinstance(info[0], status.DriveInfo)
+
+
+@pytest.mark.asyncio
+async def test_get_bandwidth_info():
+ """Bandwidth requires psutil"""
+ bandwidths = await status.get_bandwidth_info()
+ assert bandwidths == []
+
+ await status.bandwidth_worker(2)
+ bandwidths = await status.get_bandwidth_info()
+ assert isinstance(bandwidths, list)
+ assert len(bandwidths) > 0
+ assert isinstance(bandwidths[0], status.BandwidthInfo)
+
+ with mock.patch('wrolpi.status.psutil.net_io_counters') as mock_net_io_counters:
+ # NO FALLBACK!
+ mock_net_io_counters.side_effect = Exception('testing no psutil')
+ await status.bandwidth_worker(1)
+ bandwidths = await status.get_bandwidth_info()
+ assert isinstance(bandwidths[0].bytes_recv, int)
+ assert isinstance(bandwidths[0].bytes_sent, int)
+ assert isinstance(bandwidths[0].elapsed, int)
+ assert isinstance(bandwidths[0].speed, int)
+ assert isinstance(bandwidths[0].name, str)