diff --git a/.gitignore b/.gitignore index a2473f35..655c907f 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,5 @@ app/cypress/videos # Generated by release/release.sh before upload release/latest.json +perf_results/ +scripts/api_perf_hosts.json diff --git a/app/src/api.js b/app/src/api.js index a55d44a8..2d57e456 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -244,6 +244,14 @@ export async function getVideoComments(fileGroupId) { } } +export async function getVideoDescription(fileGroupId) { + const response = await apiGet(`${VIDEOS_API}/${fileGroupId}/description`); + let data = await response.json(); + return { + description: data.description, + } +} + export async function getVideoCaptions(fileGroupId) { const response = await apiGet(`${VIDEOS_API}/${fileGroupId}/captions`); let data = await response.json(); diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 86193cf2..5b643cb1 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -230,7 +230,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { // Get the Video's channel, fallback to the URL's channel id. const {channelId} = useParams(); const {channel} = useChannel(video && video.channel_id ? video.channel_id : channelId); - const {comments} = useVideoExtras(videoFile?.id); + const {comments, description: videoDescription} = useVideoExtras(videoFile?.id); const {captions, captionsLoading, fetchCaptions} = useVideoCaptions(videoFile?.id); const mediaTitle = videoFile ? (videoFile.title || videoFile.stem || videoFile.name) : null; @@ -343,10 +343,9 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { : []; let description = 'No description available.'; - if (video && video['description']) { + if (videoDescription) { // Only replace empty description if there is one available. - description = video['description']; - description = formatVideoDescription(description, setVideoTime) + description = formatVideoDescription(videoDescription, setVideoTime) } const getFile = (suffix) => videoFile.files.find(i => i.path.toLowerCase().endsWith(suffix)); diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 35a98e5d..e59f55bd 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -29,6 +29,7 @@ import { getVideo, getVideoCaptions, getVideoComments, + getVideoDescription, getVideosStatistics, postDumpConfig, postImportConfig, @@ -740,8 +741,10 @@ export const useVideo = (fileGroupId) => { } export const useVideoExtras = (fileGroupId) => { - // Fetches extra data to display on a Video's page. + // Fetches extra data to display on a Video's page. These are served separately from the + // Video itself so list responses stay small and never read files from disk. const [comments, setComments] = useState(null); + const [description, setDescription] = useState(null); const fetchComments = async () => { try { @@ -752,15 +755,26 @@ export const useVideoExtras = (fileGroupId) => { } } + const fetchDescription = async () => { + try { + const result = await getVideoDescription(fileGroupId); + setDescription(result.description); + } catch (e) { + console.error(e); + } + } + useEffect(() => { if (fileGroupId) { fetchComments(); + fetchDescription(); } else { setComments(null); + setDescription(null); } }, [fileGroupId]); - return {comments} + return {comments, description} } export const useVideoCaptions = (fileGroupId) => { diff --git a/app/src/hooks/customHooks.test.js b/app/src/hooks/customHooks.test.js index 9de8179b..7e8f8183 100644 --- a/app/src/hooks/customHooks.test.js +++ b/app/src/hooks/customHooks.test.js @@ -1,7 +1,8 @@ import React from 'react'; import {act, renderHook} from '@testing-library/react'; import {render, screen} from '../test-utils'; -import {usePages, useDriveTemperature, useDriveHealth} from './customHooks'; +import {usePages, useDriveTemperature, useDriveHealth, useVideoExtras} from './customHooks'; +import {getVideoComments, getVideoDescription} from '../api'; import {QueryContext, StatusContext} from '../contexts/contexts'; import {Paginator} from '../components/Common'; @@ -33,6 +34,12 @@ const makeWrapper = (initialParams = {}) => { return Wrapper; }; +jest.mock('../api', () => ({ + ...jest.requireActual('../api'), + getVideoComments: jest.fn(), + getVideoDescription: jest.fn(), +})); + // Mock Media so Paginator renders both mobile + tablet variants synchronously. jest.mock('../contexts/contexts', () => { const actual = jest.requireActual('../contexts/contexts'); @@ -278,3 +285,25 @@ describe('Paginator rendering', () => { expect(screen.queryByText('3')).toBeNull(); }); }); + +describe('useVideoExtras', () => { + // The description is fetched on its own, like comments: the Video payload no longer carries it, + // so list pages never pay for it. + test('fetches comments and description for a video', async () => { + getVideoComments.mockResolvedValue({comments: [{id: 'c1', parent: 'root'}]}); + getVideoDescription.mockResolvedValue({description: 'hello'}); + const {result} = renderHook(() => useVideoExtras(7)); + await act(async () => { + }); + expect(getVideoComments).toHaveBeenCalledWith(7); + expect(getVideoDescription).toHaveBeenCalledWith(7); + expect(result.current.comments).toEqual([{id: 'c1', parent: 'root'}]); + expect(result.current.description).toBe('hello'); + }); + + test('clears both when there is no video', () => { + const {result} = renderHook(() => useVideoExtras(null)); + expect(result.current.comments).toBeNull(); + expect(result.current.description).toBeNull(); + }); +}); diff --git a/modules/archive/api.py b/modules/archive/api.py index ebaa9725..e4ad570a 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -1,3 +1,4 @@ +import asyncio import multiprocessing import queue from http import HTTPStatus @@ -60,8 +61,9 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest): limit = archive_limit_limiter(body.limit) offset = body.offset or 0 - file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names, - body.headline, body.deep) + file_groups, total = await asyncio.to_thread( + lib.search_archives, search_str, domain, limit, offset, body.order_by, body.tag_names, + body.headline, body.deep) ret = dict(file_groups=file_groups, totals=dict(file_groups=total)) return json_response(ret) diff --git a/modules/docs/api.py b/modules/docs/api.py index 2aeeded3..b8f2b212 100644 --- a/modules/docs/api.py +++ b/modules/docs/api.py @@ -1,3 +1,4 @@ +import asyncio from http import HTTPStatus from sanic import Request, Blueprint @@ -46,7 +47,8 @@ doc_limit_limiter = api_param_limiter(100) async def search_docs(_: Request, body: schema.DocSearchRequest): limit = doc_limit_limiter(body.limit) offset = body.offset or 0 - file_groups, total = _search_docs( + file_groups, total = await asyncio.to_thread( + _search_docs, search_str=body.search_str, author=body.author, subject=body.subject, diff --git a/modules/videos/models.py b/modules/videos/models.py index 3320f401..22cff5a8 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -82,7 +82,6 @@ class Video(ModelHelper, Base): codec_names=codec_names, codec_types=codec_types, comments_failed=self.comments_failed, - description=self.file_group.c_text or self.get_video_description(), have_comments=self.have_comments, id=self.id, info_json_file=self.info_json_file, @@ -147,6 +146,13 @@ class Video(ModelHelper, Base): if (info_json := self.get_info_json()) and (description := info_json.get('description')): return description + def get_description(self) -> Optional[str]: + """The indexed description (`FileGroup.c_text`), falling back to the info json on disk. + + Served by its own endpoint (like comments and captions) so list responses never touch the + filesystem: `__json__` must stay free of disk reads.""" + return self.file_group.c_text or self.get_video_description() + def get_surrounding_videos(self): """ Get the previous and next videos around this Video. The videos must be in the same Channel. diff --git a/modules/videos/schema.py b/modules/videos/schema.py index 711de62e..579a6337 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -167,6 +167,11 @@ class VideoCaptionsResponse: captions: list +@dataclass +class VideoDescriptionResponse: + description: Optional[str] + + @dataclass class VideoSearchRequest: search_str: Optional[str] = None diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index fc3b8284..02f68820 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -1,3 +1,6 @@ +import asyncio +import time +from unittest import mock import json from http import HTTPStatus from pathlib import Path @@ -588,3 +591,155 @@ Another line of captions. # Assert FileGroup has correct number of files assert len(video.file_group.files) == file_count_before_caption + 1, \ f'FileGroup should have {file_count_before_caption + 1} files after adding caption' + + +@pytest.mark.asyncio +async def test_search_videos_does_not_block_event_loop(async_client): + """A slow video search must not stall other requests on the same Sanic worker. + + The search runs a synchronous SQLite query; on a large library it takes seconds cold. If it + runs on the event loop, every other request on the worker waits for it.""" + # The first request pays the test app's startup cost; keep that out of the measurement. + await async_client.get('/api/echo') + + def slow_search(*_, **__): + time.sleep(0.5) # a synchronous, blocking query + return [], 0 + + async def search(): + return await async_client.post('/api/videos/search', content=json.dumps({})) + + async def echo(): + start = time.perf_counter() + _, response = await async_client.get('/api/echo') + return response, time.perf_counter() - start + + with mock.patch('modules.videos.video.lib.search_videos', slow_search): + (_, search_response), (echo_response, echo_elapsed) = await asyncio.gather(search(), echo()) + + assert search_response.status_code == HTTPStatus.OK + assert echo_response.status_code == HTTPStatus.OK + assert echo_elapsed < 0.4, f'echo waited {echo_elapsed:.2f}s behind a blocking search' + + +@pytest.mark.asyncio +async def test_search_videos_default_page_pagination(test_session, video_factory, audio_factory, assert_video_search): + """The unfiltered Videos page (no search/tags/channel) pages through every video and audio file in + date order, including files sharing a date and files without one, and reports the true total. + + This page takes a fast path (one index range per mimetype instead of scanning every video), so + it must produce exactly what the generic query produces.""" + # Two files share a date, one has no published date at all, and both mimetypes are mixed in. + a = video_factory(title='a', upload_date='2025-01-01', with_video_file=True) + b = video_factory(title='b', upload_date='2025-01-03', with_video_file=True) + c = audio_factory(title='c', upload_date='2025-01-03', with_info_json=True) + d = video_factory(title='d', upload_date='2025-01-02', with_video_file=True) + e = audio_factory(title='e', upload_date='2025-01-04', with_info_json=True) + f = video_factory(title='f', with_video_file=True) # no upload date + test_session.commit() + f.file_group.effective_datetime = None + test_session.commit() + test_session.refresh(f.file_group) + assert f.file_group.effective_datetime is None + + from modules.videos.video import lib + with mock.patch.object(lib, '_search_videos_by_date', wraps=lib._search_videos_by_date) as fast_path: + await assert_video_search(order_by='-published_datetime', assert_total=6) + assert fast_path.called, 'the unfiltered date-ordered page should take the fast path' + fast_path.reset_mock() + await assert_video_search(order_by='-published_datetime', search_str='a', assert_total=1) + await assert_video_search(order_by='-size', assert_total=6) + assert not fast_path.called, 'searches and other orders use the generic query' + + # Newest first: NULL dates last, ties broken by id ascending. + newest = [e.id, b.id, c.id, d.id, a.id, f.id] + await assert_video_search(order_by='-published_datetime', assert_total=6, assert_ids=newest) + await assert_video_search(order_by='-published_datetime', limit=2, offset=0, assert_total=6, assert_ids=newest[0:2]) + await assert_video_search(order_by='-published_datetime', limit=2, offset=2, assert_total=6, assert_ids=newest[2:4]) + await assert_video_search(order_by='-published_datetime', limit=2, offset=4, assert_total=6, assert_ids=newest[4:6]) + # Past the end the fast path still reports the true total (the generic query has no rows to + # carry `COUNT(*) OVER()` in, so it reports 0 there; the fast path is the better behavior). + await assert_video_search(order_by='-published_datetime', limit=2, offset=6, assert_total=6, assert_ids=[]) + + # Oldest first: NULL dates first (SQLite sorts NULL lowest), ties broken by id ascending. + oldest = [f.id, a.id, d.id, b.id, c.id, e.id] + await assert_video_search(order_by='published_datetime', assert_total=6, assert_ids=oldest) + await assert_video_search(order_by='published_datetime', limit=4, offset=2, assert_total=6, assert_ids=oldest[2:6]) + + # The fast path must return exactly what the generic query returns for the same request: + # same rows, same order (NULL placement, id tiebreaker, interleaved mimetypes), same paging. + def ids(order, limit, offset): + results, total = lib.search_videos(order=order, limit=limit, offset=offset) + return [i['id'] for i in results], total + + for order in lib.INDEXED_DATE_ORDERS: + for limit, offset in ((24, 0), (2, 0), (2, 2), (4, 2), (1, 5)): + fast = ids(order, limit, offset) + with mock.patch.object(lib, 'INDEXED_DATE_ORDERS', ()): + generic = ids(order, limit, offset) + assert fast == generic, f'{order} limit={limit} offset={offset}: fast {fast} != generic {generic}' + + +def test_cached_total(test_session): + """The unfiltered Videos page total is cached briefly: counting every video is a full index scan + that costs seconds cold on a large library, and pagination can tolerate a slightly stale total.""" + from modules.videos.video import lib + calls = [] + + def compute(): + calls.append(1) + return 42 + + lib._TOTALS_CACHE.clear() + with mock.patch('modules.videos.video.lib.PYTEST', False): + assert lib._cached_total('k', compute) == 42 + assert lib._cached_total('k', compute) == 42 + assert len(calls) == 1, 'second call within the TTL should hit the cache' + # Expire the entry. + cached_at, total = lib._TOTALS_CACHE['k'] + lib._TOTALS_CACHE['k'] = (cached_at - lib.TOTALS_CACHE_SECONDS - 1, total) + assert lib._cached_total('k', compute) == 42 + assert len(calls) == 2, 'expired entry should be recomputed' + + # Tests need exact totals, so the cache is bypassed under pytest. + assert lib._cached_total('k', compute) == 42 + assert len(calls) == 3 + lib._TOTALS_CACHE.clear() + +async def test_video_description_endpoint(async_client, test_session, video_factory): + """The description is served by its own endpoint, like comments and captions. It comes from the + indexed `c_text`, falling back to the info json on disk for a video that was never indexed with one.""" + video = video_factory(with_video_file=True, with_info_json={'description': 'from the info json'}) + test_session.commit() + assert video.file_group.c_text == 'from the info json', 'indexing copies the description into c_text' + + request, response = await async_client.get(f'/api/videos/{video.file_group_id}/description') + assert response.status_code == HTTPStatus.OK + assert response.json == {'description': 'from the info json'} + + # A stale row (indexed before descriptions were stored) falls back to the file. + video.file_group.c_text = None + test_session.commit() + request, response = await async_client.get(f'/api/videos/{video.file_group_id}/description') + assert response.status_code == HTTPStatus.OK + assert response.json == {'description': 'from the info json'} + + request, response = await async_client.get('/api/videos/123456/description') + assert response.status_code == HTTPStatus.NOT_FOUND + + +@pytest.mark.asyncio +async def test_video_json_does_not_read_info_json(async_client, test_session, video_factory, assert_video_search): + """Video list and detail responses must never read the info json from disk. + + Many videos have no description, so a fallback read from `__json__` would open and parse every + such video's info json on every list page, cold, for a field the list never displays.""" + video = video_factory(with_video_file=True, with_info_json={'duration': 5}) # no description + test_session.commit() + assert not video.file_group.c_text + + with mock.patch('modules.videos.models.Video.get_info_json', side_effect=AssertionError('read info json')): + await assert_video_search(assert_total=1, assert_ids=[video.id]) + request, response = await async_client.get(f'/api/videos/{video.file_group_id}') + assert response.status_code == HTTPStatus.OK + assert 'description' not in response.json['file_group']['video'] diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index 4a4b96b6..ac1bc60f 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -1,3 +1,4 @@ +import asyncio from http import HTTPStatus from sanic import Blueprint @@ -36,6 +37,15 @@ def video_get_comments(_: Request, file_group_id: int): return json_response({'comments': video.get_comments()}) +@video_bp.get('//description') +@openapi.description('Get Video description') +@openapi.response(HTTPStatus.OK, schema.VideoDescriptionResponse) +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +def video_get_description(_: Request, file_group_id: int): + video = lib.get_video(file_group_id) + return json_response({'description': video.get_description()}) + + @video_bp.get('//captions') @openapi.description('Get Video captions') @openapi.response(HTTPStatus.OK, schema.VideoCaptionsResponse) @@ -57,7 +67,9 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest): if body.order_by not in lib.VIDEO_ORDERS: raise InvalidOrderBy('Invalid order by') - file_groups, videos_total = lib.search_videos( + # Synchronous SQLite query; keep it off the event loop. + file_groups, videos_total = await asyncio.to_thread( + lib.search_videos, body.search_str, body.offset, body.limit, diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index b497af9b..718c684f 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -1,7 +1,8 @@ import asyncio import random +import time from datetime import timedelta -from typing import Tuple, Optional, List +from typing import Tuple, Optional, List, Dict, Callable import yt_dlp from sqlalchemy import or_ @@ -11,12 +12,12 @@ from modules.videos.models import Video, Channel from wrolpi import fts from wrolpi.common import logger, limit_concurrent, wrol_mode_check from wrolpi.dates import now -from wrolpi.db import get_db_session +from wrolpi.db import get_db_session, get_db_curs from wrolpi.downloader import download_manager from wrolpi.files.lib import handle_file_group_search_results from wrolpi.files.models import FileGroup from wrolpi.tags import tag_append_sub_select_where -from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR +from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR, PYTEST from ..cookies import cookies_unlocked, cookies_for_download from ..lib import get_yt_dlp_http_headers, get_yt_dlp_sleep_opts @@ -86,6 +87,82 @@ JOIN_ORDERS = ('view_count', '-view_count') DEFAULT_VIDEO_ORDER = 'rank' VIDEO_QUERY_LIMIT = 24 +# Orders the unfiltered Videos page can serve straight from the (mimetype, effective_datetime) index. +INDEXED_DATE_ORDERS = ('published_datetime', '-published_datetime') +VIDEO_MIMETYPE_PREFIXES = ('video/', 'audio/') + +# The unfiltered Videos page total, cached for a short time. Counting every video walks the whole +# mimetype index, which costs seconds cold on a large library on a spinning disk; pagination can +# tolerate a total that is a minute stale. Bypassed under pytest so tests see exact totals. +TOTALS_CACHE_SECONDS = 60 +_TOTALS_CACHE: Dict[str, Tuple[float, int]] = dict() + + +def _cached_total(key: str, compute: Callable[[], int]) -> int: + if not PYTEST: + hit = _TOTALS_CACHE.get(key) + if hit and time.monotonic() - hit[0] < TOTALS_CACHE_SECONDS: + return hit[1] + total = compute() + _TOTALS_CACHE[key] = (time.monotonic(), total) + return total + + +def _video_mimetypes() -> List[str]: + """Every distinct video/* and audio/* mimetype in the library (a handful; served by the mimetype index).""" + mimetypes = [] + with get_db_curs() as curs: + for prefix in VIDEO_MIMETYPE_PREFIXES: + # '0' is the character after '/', so this is the half-open range of the prefix. + curs.execute('SELECT DISTINCT mimetype FROM file_group WHERE mimetype >= ? AND mimetype < ?', + (prefix, prefix[:-1] + '0')) + mimetypes.extend(i[0] for i in curs.fetchall()) + return mimetypes + + +def _count_videos() -> int: + ranges = ' OR '.join(f'(mimetype >= :p{i} AND mimetype < :e{i})' for i in range(len(VIDEO_MIMETYPE_PREFIXES))) + params = dict() + for i, prefix in enumerate(VIDEO_MIMETYPE_PREFIXES): + params[f'p{i}'], params[f'e{i}'] = prefix, prefix[:-1] + '0' + with get_db_curs() as curs: + curs.execute(f'SELECT COUNT(*) FROM file_group WHERE {ranges}', params) + return curs.fetchone()[0] + + +def _search_videos_by_date(order: str, limit: int, offset: int) -> Tuple[List[dict], int]: + """The unfiltered Videos page, ordered by date, without visiting every video. + + The generic query (`mimetype LIKE 'video/%' OR ...` + `COUNT(*) OVER()`) must visit every + video's index entry before it can apply LIMIT, which scales with the library rather than the + page. Here each exact mimetype is a pure covering-index seek on (mimetype, effective_datetime) + that reads only `limit + offset` entries; those few rows are then merged. Results must be + identical to the generic query's (same ORDER BY, including the id tiebreaker and NULL placement). + The total is a separate, cached count.""" + order_by = VIDEO_ORDERS[order] + mimetypes = _video_mimetypes() + if not mimetypes: + return [], 0 + + n = limit + offset + params = {f'mt{i}': mimetype for i, mimetype in enumerate(mimetypes)} + params['offset'] = offset + members = '\n UNION ALL\n'.join( + f'SELECT * FROM (SELECT fg.id, fg.effective_datetime FROM file_group fg' + f' WHERE fg.mimetype = :mt{i} ORDER BY {order_by} LIMIT {n})' + for i in range(len(mimetypes)) + ) + stmt = f''' + SELECT fg.id + FROM ({members}) fg + ORDER BY {order_by} + LIMIT {limit} OFFSET :offset + '''.strip() + logger.debug(f'{stmt} {params}') + + total = _cached_total('videos', _count_videos) + return handle_file_group_search_results(stmt, params, total=total) + def search_videos( search_str: str = None, @@ -109,7 +186,12 @@ def search_videos( # Explicit JSON nulls bypass the schema defaults; SQLite rejects LIMIT/OFFSET NULL. limit = int(limit) if limit else 20 - params = dict(search_str=search_str, offset=int(offset) if offset else 0) + offset = int(offset) if offset else 0 + if not search_str and not tag_names and not channel_id and not censored and order in INDEXED_DATE_ORDERS: + # The Videos page with no filters, in date order. + return _search_videos_by_date(order, limit, offset) + + params = dict(search_str=search_str, offset=offset) if channel_id: wheres.append('v.channel_id = :channel_id') joins.append('LEFT JOIN channel c ON c.id = v.channel_id') diff --git a/modules/zim/api.py b/modules/zim/api.py index bd804531..2aaf240b 100644 --- a/modules/zim/api.py +++ b/modules/zim/api.py @@ -1,3 +1,4 @@ +import asyncio import urllib.parse from http import HTTPStatus @@ -8,6 +9,7 @@ from sanic_ext.extensions.openapi import openapi from wrolpi import lang from wrolpi.api_utils import json_response from wrolpi.common import logger +from wrolpi.db import get_db_session from wrolpi.downloader import download_manager from wrolpi.events import Events from . import lib, schema @@ -63,10 +65,15 @@ async def post_set_zim_auto_search(request: Request, zim_id: int, body: schema.Z body=schema.ZimSearchRequest, ) @validate(schema.ZimSearchRequest) -async def search_all_zims(request: Request, body: schema.ZimSearchRequest): - session = request.ctx.session - results = lib.search_all_zims(session, body.search_str, tag_names=body.tag_names, offset=body.offset, - limit=body.limit) +async def search_all_zims(_: Request, body: schema.ZimSearchRequest): + # libzim searches are synchronous and slow; run off the event loop. The session is opened in the + # worker thread: a SQLAlchemy Session must not be shared across threads. + def search(): + with get_db_session() as session: + return lib.search_all_zims(session, body.search_str, tag_names=body.tag_names, offset=body.offset, + limit=body.limit) + + results = await asyncio.to_thread(search) return json_response({'zims': results}) @@ -77,10 +84,13 @@ async def search_all_zims(request: Request, body: schema.ZimSearchRequest): response=schema.ZimSearchResponse, ) @validate(schema.ZimSearchRequest) -async def search_zim(request: Request, zim_id: int, body: schema.ZimSearchRequest): - session = request.ctx.session - headlines = lib.headline_zim(session, body.search_str, zim_id, tag_names=body.tag_names, offset=body.offset, - limit=body.limit) +async def search_zim(_: Request, zim_id: int, body: schema.ZimSearchRequest): + def search(): + with get_db_session() as session: + return lib.headline_zim(session, body.search_str, zim_id, tag_names=body.tag_names, offset=body.offset, + limit=body.limit) + + headlines = await asyncio.to_thread(search) return json_response({'zim': headlines}) @@ -205,26 +215,20 @@ async def post_search_estimates(request: Request, body: schema.SearchEstimateReq if not body.search_str and not body.tag_names: return response.empty(HTTPStatus.BAD_REQUEST) - session = request.ctx.session + def estimate(): + zims_estimates = list() + with get_db_session() as session: + if body.tag_names: + # Get actual count of entries tagged with the tag names. + counts = Zims.entries_with_tags(session, body.tag_names) + else: + # Get estimates using libzim. + counts = Zims.estimate(session, body.search_str) + for zim, count in counts.items(): + zims_estimates.append(dict(estimate=count, **zim.__json__())) + return zims_estimates - if body.tag_names: - # Get actual count of entries tagged with the tag names. - zims_estimates = list() - for zim, count in Zims.entries_with_tags(session, body.tag_names).items(): - d = dict( - estimate=count, - **zim.__json__(), - ) - zims_estimates.append(d) - else: - # Get estimates using libzim. - zims_estimates = list() - for zim, estimate in Zims.estimate(session, body.search_str).items(): - d = dict( - estimate=estimate, - **zim.__json__(), - ) - zims_estimates.append(d) + zims_estimates = await asyncio.to_thread(estimate) ret = dict( zims_estimates=zims_estimates diff --git a/scripts/api_perf.py b/scripts/api_perf.py new file mode 100644 index 00000000..579aca57 --- /dev/null +++ b/scripts/api_perf.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +""" +Benchmark the read-only WROLPi API endpoints, hot and cold. + +Read-only here means "does not mutate the library": every GET endpoint, plus the POST +`.../search` and `.../estimates` endpoints the React UI uses to render its pages (the Videos +page, for example, is rendered from `POST /api/videos/search`). + +Usage examples: + + # Hot only (no SSH needed), 3 passes, against a host profile from scripts/api_perf_hosts.json + # (copy scripts/api_perf_hosts.example.json to create it). + python3 scripts/api_perf.py --host docker + + # Cold pass (restart the API over SSH, then run every endpoint once) followed by 3 hot passes. + python3 scripts/api_perf.py --host docker --cold + + # Additionally restart before EACH of the "page" endpoints so each gets a truly cold number. + python3 scripts/api_perf.py --host decay --cold --cold-each + + # Ad-hoc host. + python3 scripts/api_perf.py --url https://192.0.2.10:8443 + +Results are written as CSV to ./perf_results/_.csv, one row per request, and a +summary table is printed at the end. Only the standard library is used so this runs anywhere. +""" +import argparse +import csv +import datetime +import json +import shlex +import ssl +import statistics +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Optional + +# ---------------------------------------------------------------------------------------------- +# Hosts +# ---------------------------------------------------------------------------------------------- + +HOSTS_FILE = Path(__file__).with_name('api_perf_hosts.json') # gitignored; see api_perf_hosts.example.json + + +def load_hosts() -> dict: + """Host profiles live in a gitignored JSON file so LAN addresses/usernames stay out of the repo. + + Each profile: url, ssh (user@host or null), ssh_opts (list), restart (shell command run over + ssh), drop_caches (shell command or null when the ssh user cannot do it).""" + if not HOSTS_FILE.exists(): + return {} + return json.loads(HOSTS_FILE.read_text()) + + +HOSTS = load_hosts() + +# ---------------------------------------------------------------------------------------------- +# Endpoint catalog +# ---------------------------------------------------------------------------------------------- +# Each entry: name, method, path, optional json body, optional `page` (which UI page issues it on +# load; used for grouping and for --cold-each), optional `needs` (an id discovered at runtime, +# substituted into the path as {video_id}, etc.). +# +# Deliberately excluded (read-only but not useful to time, or slow by design): +# GET /api/events/feed (long-poll), GET /api/files/zip/download (streams a zip), +# GET /api/extensions/ (static file), GET /api/upgrade/check (network), +# GET /api/ai/services//logs, GET /api/ai/files/read, GET /api/ai/help/, +# GET /api/echo (used as the readiness probe instead). + +CATALOG = [ + # --- root --- + dict(name='echo', method='GET', path='/api/echo'), + dict(name='settings', method='GET', path='/api/settings', page='app'), + dict(name='status', method='GET', path='/api/status', page='app'), + dict(name='statistics', method='GET', path='/api/statistics', page='home'), + dict(name='downloaders', method='GET', path='/api/downloaders'), + dict(name='download', method='GET', path='/api/download', page='downloads'), + dict(name='tag', method='GET', path='/api/tag', page='app'), + dict(name='tag/recent', method='GET', path='/api/tag/recent'), + dict(name='extensions', method='GET', path='/api/extensions'), + dict(name='search_suggestions', method='POST', path='/api/search_suggestions', + body={'search_str': '{q}'}, page='search'), + dict(name='search_file_estimates', method='POST', path='/api/search_file_estimates', + body={'search_str': '{q}'}, page='search'), + dict(name='search_other_estimates', method='POST', path='/api/search_other_estimates', + body={'tag_names': []}, page='search'), + + # --- config --- + dict(name='config', method='GET', path='/api/config/'), + dict(name='config/backups', method='GET', path='/api/config/backups'), + + # --- files --- + dict(name='files/refresh_progress', method='GET', path='/api/files/refresh_progress', page='files'), + dict(name='files/worker_status', method='GET', path='/api/files/worker_status'), + dict(name='files/bulk_tag/progress', method='GET', path='/api/files/bulk_tag/progress'), + dict(name='files (root listing)', method='POST', path='/api/files/', body={'directories': []}, page='files'), + dict(name='files/search (no query)', method='POST', path='/api/files/search', + body={'offset': 0, 'limit': 20}, page='search'), + dict(name='files/search (query)', method='POST', path='/api/files/search', + body={'offset': 0, 'limit': 20, 'search_str': '{q}'}, page='search'), + dict(name='files/search (common word "the")', method='POST', path='/api/files/search', + body={'offset': 0, 'limit': 20, 'search_str': 'the'}), + dict(name='files/search (query, deep)', method='POST', path='/api/files/search', + body={'offset': 0, 'limit': 20, 'search_str': '{q}', 'deep': True}), + dict(name='files/search (page 50)', method='POST', path='/api/files/search', + body={'offset': 1000, 'limit': 20, 'search_str': '{q}'}), + dict(name='files/search_directories', method='POST', path='/api/files/search_directories', + body={'path': 'vid'}), + + # --- collections --- + dict(name='collections', method='GET', path='/api/collections/', page='collections'), + dict(name='collections/', method='GET', path='/api/collections/{collection_id}', needs='collection_id'), + dict(name='collections/search', method='POST', path='/api/collections/search', body={}), + dict(name='collections/reorganize/channels', method='GET', path='/api/collections/reorganize/channels'), + dict(name='collections/reorganize/domains', method='GET', path='/api/collections/reorganize/domains'), + + # --- videos --- + dict(name='videos/statistics', method='GET', path='/api/videos/statistics', page='videos-stats'), + dict(name='videos/download-defaults', method='GET', path='/api/videos/download-defaults'), + dict(name='videos/cookies/status', method='GET', path='/api/videos/cookies/status'), + dict(name='videos/suggested-user-agent', method='GET', path='/api/videos/suggested-user-agent'), + dict(name='videos/channels', method='GET', path='/api/videos/channels', page='videos'), + dict(name='videos/channels/', method='GET', path='/api/videos/channels/{channel_id}', + needs='channel_id', page='channel'), + dict(name='videos/channels/search', method='POST', path='/api/videos/channels/search', body={'tag_names': []}), + dict(name='videos/search (page 1, newest)', method='POST', path='/api/videos/search', + body={'offset': 0, 'limit': 24, 'order_by': '-published_datetime'}, page='videos'), + dict(name='videos/search (page 1, -view_count)', method='POST', path='/api/videos/search', + body={'offset': 0, 'limit': 24, 'order_by': '-view_count'}), + dict(name='videos/search (page 50, newest)', method='POST', path='/api/videos/search', + body={'offset': 1176, 'limit': 24, 'order_by': '-published_datetime'}), + dict(name='videos/search (query, rank)', method='POST', path='/api/videos/search', + body={'offset': 0, 'limit': 24, 'order_by': 'rank', 'search_str': '{q}'}, page='videos-search'), + dict(name='videos/search (common word "the")', method='POST', path='/api/videos/search', + body={'offset': 0, 'limit': 24, 'order_by': 'rank', 'search_str': 'the'}), + dict(name='videos/search (query, deep)', method='POST', path='/api/videos/search', + body={'offset': 0, 'limit': 24, 'order_by': 'rank', 'search_str': '{q}', 'deep': True}), + dict(name='videos/search (channel)', method='POST', path='/api/videos/search', + body={'offset': 0, 'limit': 24, 'order_by': '-published_datetime', 'channel_id': '{channel_id}'}, + needs='channel_id', page='channel'), + dict(name='videos/', method='GET', path='/api/videos/{video_id}', needs='video_id', page='video'), + dict(name='videos//comments', method='GET', path='/api/videos/{video_id}/comments', needs='video_id', + page='video'), + dict(name='videos//captions', method='GET', path='/api/videos/{video_id}/captions', needs='video_id'), + + # --- archive --- + dict(name='archive/statistics', method='GET', path='/api/archive/statistics', page='archives-stats'), + dict(name='archive/browsers', method='GET', path='/api/archive/browsers'), + dict(name='archive/search (page 1)', method='POST', path='/api/archive/search', + body={'offset': 0, 'limit': 20, 'order_by': '-published_datetime'}, page='archives'), + dict(name='archive/search (query)', method='POST', path='/api/archive/search', + body={'offset': 0, 'limit': 20, 'search_str': '{q}'}, page='archives-search'), + dict(name='archive/search (query, deep)', method='POST', path='/api/archive/search', + body={'offset': 0, 'limit': 20, 'search_str': '{q}', 'deep': True}), + dict(name='archive/', method='GET', path='/api/archive/{archive_id}', needs='archive_id', page='archive'), + + # --- docs --- + dict(name='docs/statistics', method='GET', path='/api/docs/statistics'), + dict(name='docs/search (page 1)', method='POST', path='/api/docs/search', body={'limit': 20}, page='docs'), + dict(name='docs/search (query)', method='POST', path='/api/docs/search', body={'limit': 20, 'search_str': '{q}'}), + dict(name='docs/', method='GET', path='/api/docs/{doc_id}', needs='doc_id'), + + # --- zim --- + dict(name='zim', method='GET', path='/api/zim/', page='zim'), + dict(name='zim/subscribe', method='GET', path='/api/zim/subscribe'), + dict(name='zim/outdated', method='GET', path='/api/zim/outdated'), + dict(name='zim/search', method='POST', path='/api/zim/search', body={'search_str': '{q}', 'limit': 10}, + page='search'), + dict(name='zim/search_estimates', method='POST', path='/api/zim/search_estimates', + body={'search_str': '{q}'}, page='search'), + + # --- map --- + dict(name='map/files', method='GET', path='/api/map/files', page='map'), + dict(name='map/subscribe', method='GET', path='/api/map/subscribe'), + dict(name='map/pins', method='GET', path='/api/map/pins', page='map'), + dict(name='map/search', method='GET', path='/api/map/search?q={q}&limit=20&offset=0'), + dict(name='map/search/estimate', method='GET', path='/api/map/search/estimate?q={q}'), + dict(name='map/search/status', method='GET', path='/api/map/search/status'), + + # --- inventory --- + dict(name='inventory', method='GET', path='/api/inventory/', page='inventory'), + dict(name='inventory/catalog', method='GET', path='/api/inventory/catalog'), + + # --- flasher --- + dict(name='flasher/configs', method='GET', path='/api/flasher/configs'), + dict(name='flasher/search', method='POST', path='/api/flasher/search', body={}), + + # --- ai (MCP-facing read endpoints) --- + dict(name='ai/status', method='GET', path='/api/ai/status'), + dict(name='ai/services', method='GET', path='/api/ai/services'), + dict(name='ai/disks', method='GET', path='/api/ai/disks'), + dict(name='ai/manage/catalog', method='GET', path='/api/ai/manage/catalog'), + dict(name='ai/collections', method='GET', path='/api/ai/collections'), + dict(name='ai/inventories', method='GET', path='/api/ai/inventories'), + dict(name='ai/zims', method='GET', path='/api/ai/zims'), + dict(name='ai/search', method='POST', path='/api/ai/search', body={'search_str': '{q}', 'limit': 5}), + dict(name='ai/videos/search', method='POST', path='/api/ai/videos/search', body={'search_str': '{q}', 'limit': 5}), + dict(name='ai/archives/search', method='POST', path='/api/ai/archives/search', + body={'search_str': '{q}', 'limit': 5}), + dict(name='ai/docs/search', method='POST', path='/api/ai/docs/search', body={'search_str': '{q}', 'limit': 5}), + dict(name='ai/zims/search', method='POST', path='/api/ai/zims/search', body={'search_str': '{q}', 'limit': 5}), + dict(name='ai/videos/', method='GET', path='/api/ai/videos/{video_id}', needs='video_id'), + dict(name='ai/archives/', method='GET', path='/api/ai/archives/{archive_id}', needs='archive_id'), + dict(name='chat/modes', method='GET', path='/api/chat/modes'), +] + +# Endpoints that get their own restart when --cold-each is used: what a user hits when they open +# a page. Keep this short: each entry costs a restart + readiness wait. +COLD_EACH = [ + 'videos/search (page 1, newest)', + 'videos/channels', + 'videos/statistics', + 'files/search (no query)', + 'files/search (query)', + 'archive/search (page 1)', + 'statistics', + 'status', + 'settings', +] + +# ---------------------------------------------------------------------------------------------- +# HTTP +# ---------------------------------------------------------------------------------------------- + +SSL_CTX = ssl.create_default_context() +SSL_CTX.check_hostname = False +SSL_CTX.verify_mode = ssl.CERT_NONE + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + # While the API is down, Caddy 302s /api/* to the Controller's fallback UI, which answers 200 + # with HTML. Following that would make a down API look like a fast one. + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +OPENER = urllib.request.build_opener(_NoRedirect, urllib.request.HTTPSHandler(context=SSL_CTX)) + + +def request(base_url: str, method: str, path: str, body=None, timeout: float = 180.0) -> dict: + """Perform one request, reading the whole body. Returns timing + status + size.""" + url = base_url.rstrip('/') + path + data = None + headers = {'Accept': 'application/json'} + if body is not None: + data = json.dumps(body).encode() + headers['Content-Type'] = 'application/json' + req = urllib.request.Request(url, data=data, method=method, headers=headers) + start = time.perf_counter() + status, size, error, payload, ctype = None, 0, None, None, '' + try: + with OPENER.open(req, timeout=timeout) as resp: + payload = resp.read() + status = resp.status + size = len(payload) + ctype = resp.headers.get('Content-Type', '') + except urllib.error.HTTPError as e: + status = e.code + try: + payload = e.read() + size = len(payload) + except Exception: + pass + error = f'HTTP {e.code}' + except Exception as e: # timeouts, connection refused, etc. + error = type(e).__name__ + (f': {e}' if str(e) else '') + elapsed = time.perf_counter() - start + if status == 200 and 'json' not in ctype: + # A 200 that is not JSON did not come from the API (fallback UI, proxy page, ...). + status, error = 'HTML', f'non-JSON 200 ({ctype.split(";")[0] or "no content-type"})' + return dict(ms=round(elapsed * 1000, 1), status=status, bytes=size, error=error, payload=payload) + + +def wait_ready(base_url: str, timeout: float = 300.0) -> float: + """Poll /api/echo until it answers 200. Returns seconds waited.""" + start = time.perf_counter() + while time.perf_counter() - start < timeout: + r = request(base_url, 'GET', '/api/echo', timeout=5) + if r['status'] == 200: + return round(time.perf_counter() - start, 1) + time.sleep(1) + raise TimeoutError(f'{base_url} did not become ready within {timeout}s') + + +# ---------------------------------------------------------------------------------------------- +# SSH / cold +# ---------------------------------------------------------------------------------------------- + +def ssh(host: dict, cmd: str, check: bool = True) -> subprocess.CompletedProcess: + argv = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', *host['ssh_opts'], host['ssh'], cmd] + proc = subprocess.run(argv, capture_output=True, text=True) + if check and proc.returncode != 0: + raise RuntimeError(f'ssh {host["ssh"]} {shlex.quote(cmd)} failed rc={proc.returncode}: {proc.stderr.strip()}') + return proc + + +def go_cold(host: dict, log) -> dict: + """Restart the API (and drop the OS page cache when we can), then wait for readiness.""" + dropped = False + if host.get('drop_caches'): + proc = ssh(host, host['drop_caches'], check=False) + dropped = proc.returncode == 0 + if not dropped: + log(f' ! drop_caches failed (continuing, process-cold only): {proc.stderr.strip()}') + t = time.perf_counter() + ssh(host, host['restart']) + # Give the old process a moment to die so we don't get a "ready" from the old process. + time.sleep(2) + ready = wait_ready(host['url']) + total = round(time.perf_counter() - t, 1) + log(f' restarted (drop_caches={dropped}); API ready after {ready}s (restart+ready {total}s)') + return dict(dropped_caches=dropped, ready_seconds=ready) + + +# ---------------------------------------------------------------------------------------------- +# Discovery of ids +# ---------------------------------------------------------------------------------------------- + +def _first_id(payload: Optional[bytes], *keys) -> Optional[int]: + if not payload: + return None + try: + data = json.loads(payload) + except Exception: + return None + for key in keys: + data = data.get(key) if isinstance(data, dict) else None + if data is None: + return None + if isinstance(data, list) and data: + item = data[0] + return item.get('id') if isinstance(item, dict) else None + return None + + +def discover_ids(base_url: str, log) -> dict: + ids = {} + + def probe(key, method, path, body, *keys): + r = request(base_url, method, path, body) + ids[key] = _first_id(r['payload'], *keys) + log(f' [discover ] {r["ms"]:9.1f} ms {r["status"]} {r["bytes"]:>9} B {path}' + + (f' <-- {r["error"]}' if r['error'] else '')) + + probe('video_id', 'POST', '/api/videos/search', {'offset': 0, 'limit': 1, 'order_by': '-published_datetime'}, + 'file_groups') + probe('archive_id', 'POST', '/api/archive/search', {'offset': 0, 'limit': 1}, 'file_groups') + probe('doc_id', 'POST', '/api/docs/search', {'limit': 1}, 'file_groups') + probe('channel_id', 'GET', '/api/videos/channels', None, 'channels') + probe('collection_id', 'GET', '/api/collections/', None, 'collections') + log(f' discovered ids: {ids}') + return ids + + +def resolve(entry: dict, ids: dict) -> Optional[dict]: + """Substitute discovered ids (and the {q} search term) into an entry. + Returns None when a required id is missing.""" + needs = entry.get('needs') + if needs and ids.get(needs) is None: + return None + e = dict(entry) + q = ids.get('q', 'solar') + e['path'] = e['path'].replace('{q}', urllib.parse.quote(q)) + if e.get('body'): + e['body'] = {bk: (q if bv == '{q}' else bv) for bk, bv in e['body'].items()} + for k, v in ids.items(): + if v is None or k == 'q': + continue + e['path'] = e['path'].replace('{%s}' % k, str(v)) + if e.get('body'): + e['body'] = {bk: (v if bv == '{%s}' % k else bv) for bk, bv in e['body'].items()} + return e + + +# ---------------------------------------------------------------------------------------------- +# Runner +# ---------------------------------------------------------------------------------------------- + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--host', choices=sorted(HOSTS), help=f'A host profile from {HOSTS_FILE.name}.') + g.add_argument('--url', help='Ad-hoc base URL, e.g. https://192.0.2.10:8443 (hot only unless --ssh given).') + ap.add_argument('--cold', action='store_true', help='Restart the API over SSH, run one cold pass first.') + ap.add_argument('--cold-each', action='store_true', + help='Also restart before each COLD_EACH endpoint for per-endpoint cold numbers.') + ap.add_argument('--hot-passes', type=int, default=3, help='Number of hot passes (default 3).') + ap.add_argument('--query', default='solar', + help='Search term substituted for {q} in the catalog (default "solar"; "the" rows are fixed worst-case).') + ap.add_argument('--only', help='Substring filter on endpoint names.') + ap.add_argument('--timeout', type=float, default=180.0, help='Per-request timeout in seconds.') + ap.add_argument('--out-dir', default='perf_results') + args = ap.parse_args() + + if args.host: + host = dict(HOSTS[args.host], name=args.host) + else: + host = dict(url=args.url, name=args.url.replace('https://', '').replace('http://', '').replace(':', '_'), + ssh=None, ssh_opts=[], restart=None, drop_caches=None) + if args.cold and not host.get('ssh'): + ap.error('--cold requires a --host profile with SSH access') + host.setdefault('ssh_opts', []) + + base = host['url'] + stamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + out_dir = Path(args.out_dir) + out_dir.mkdir(exist_ok=True) + out_csv = out_dir / f'{host["name"]}_{stamp}.csv' + log_path = out_dir / f'{host["name"]}_{stamp}.log' + log_file = open(log_path, 'w') + + def log(msg=''): + print(msg, flush=True) + log_file.write(msg + '\n') + log_file.flush() + + log(f'== WROLPi API perf: {host["name"]} ({base}) at {stamp}') + ready = wait_ready(base, timeout=30) + log(f' reachable (echo answered after {ready}s)') + + ids = discover_ids(base, log) + ids['q'] = args.query + log(f' query term: {args.query!r}') + catalog = [e for e in (resolve(e, ids) for e in CATALOG) if e] + skipped = [e['name'] for e in CATALOG if not resolve(e, ids)] + if args.only: + catalog = [e for e in catalog if args.only in e['name']] + if skipped: + log(f' skipped (no id on this host): {skipped}') + log(f' {len(catalog)} endpoints') + + rows = [] + writer = csv.DictWriter(open(out_csv, 'w', newline=''), + fieldnames=['host', 'phase', 'pass', 'name', 'method', 'path', 'ms', 'status', 'bytes', + 'error', 'dropped_caches', 'ready_seconds']) + writer.writeheader() + + def run_pass(phase: str, pass_no: int, entries, extra=None): + for e in entries: + r = request(base, e['method'], e['path'], e.get('body'), timeout=args.timeout) + row = dict(host=host['name'], phase=phase, **{'pass': pass_no}, name=e['name'], method=e['method'], + path=e['path'], ms=r['ms'], status=r['status'], bytes=r['bytes'], error=r['error'] or '', + **(extra or {})) + rows.append(row) + writer.writerow(row) + flag = '' if r['status'] == 200 else f' <-- {r["error"]}' + log(f' [{phase:9}] {r["ms"]:9.1f} ms {r["status"]} {r["bytes"]:>9} B {e["name"]}{flag}') + + # --- cold-each: restart before each key endpoint --- + if args.cold and args.cold_each: + log('\n-- cold-each: restart before each page endpoint') + by_name = {e['name']: e for e in catalog} + for name in COLD_EACH: + if name not in by_name: + continue + info = go_cold(host, log) + run_pass('cold-each', 0, [by_name[name]], extra=info) + + # --- cold pass: one restart, then every endpoint once, in catalog order --- + if args.cold: + log('\n-- cold pass: restart, then every endpoint once') + info = go_cold(host, log) + run_pass('cold', 0, catalog, extra=info) + + # --- hot passes --- + for i in range(1, args.hot_passes + 1): + log(f'\n-- hot pass {i}/{args.hot_passes}') + run_pass('hot', i, catalog) + + # --- summary --- + log(f'\n== Summary for {host["name"]} ({base}) [ms]') + header = f'{"endpoint":42} {"cold-each":>9} {"cold":>9} {"hot min":>9} {"hot med":>9} {"hot max":>9} {"status":>6} {"bytes":>9}' + log(header) + log('-' * len(header)) + summary = [] + for e in catalog: + name = e['name'] + ce = [r['ms'] for r in rows if r['name'] == name and r['phase'] == 'cold-each'] + cold = [r['ms'] for r in rows if r['name'] == name and r['phase'] == 'cold'] + hot = [r['ms'] for r in rows if r['name'] == name and r['phase'] == 'hot'] + statuses = {r['status'] for r in rows if r['name'] == name} + size = max((r['bytes'] for r in rows if r['name'] == name), default=0) + fmt = lambda v: f'{v:9.0f}' if v is not None else f'{"-":>9}' + hot_min = min(hot) if hot else None + hot_med = statistics.median(hot) if hot else None + hot_max = max(hot) if hot else None + summary.append(dict(name=name, cold_each=ce[0] if ce else None, cold=cold[0] if cold else None, + hot_min=hot_min, hot_med=hot_med, hot_max=hot_max, status=statuses, bytes=size)) + st = ','.join(str(s) for s in sorted(statuses, key=str)) + log(f'{name:42} {fmt(ce[0] if ce else None)} {fmt(cold[0] if cold else None)} {fmt(hot_min)} ' + f'{fmt(hot_med)} {fmt(hot_max)} {st:>6} {size:>9}') + + # Slowest endpoints, by hot median and by cold. + def top(key, n=10): + vals = [s for s in summary if s[key] is not None] + return sorted(vals, key=lambda s: -s[key])[:n] + + log('\n-- slowest by hot median') + for s in top('hot_med'): + log(f' {s["hot_med"]:9.0f} ms {s["name"]}') + if any(s['cold'] is not None for s in summary): + log('\n-- slowest cold') + for s in top('cold'): + log(f' {s["cold"]:9.0f} ms {s["name"]}') + + # Page-load view: sum of the endpoints a page fires when opened (they run in parallel in the + # browser, so the max is the floor and the sum is the ceiling of what the user waits). + pages = {} + for e in catalog: + if e.get('page'): + pages.setdefault(e['page'], []).append(e['name']) + log('\n-- page load (hot median / cold): max = parallel floor, sum = serial ceiling') + for page, names in sorted(pages.items()): + ss = [s for s in summary if s['name'] in names] + hm = [s['hot_med'] for s in ss if s['hot_med'] is not None] + cd = [s['cold'] for s in ss if s['cold'] is not None] + hot_txt = f'hot max {max(hm):7.0f} sum {sum(hm):7.0f}' if hm else 'hot -' + cold_txt = f'cold max {max(cd):7.0f} sum {sum(cd):7.0f}' if cd else '' + log(f' {page:16} {hot_txt} {cold_txt} ({len(names)} calls)') + + log(f'\nrows: {out_csv}\nlog: {log_path}') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/api_perf_compare.py b/scripts/api_perf_compare.py new file mode 100644 index 00000000..411b66a9 --- /dev/null +++ b/scripts/api_perf_compare.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Merge the CSVs written by scripts/api_perf.py into one cross-host table. + + python3 scripts/api_perf_compare.py perf_results/*.csv + python3 scripts/api_perf_compare.py perf_results/*.csv --sort hot --top 25 + python3 scripts/api_perf_compare.py perf_results/*.csv --markdown > perf_results/summary.md + +For each endpoint and host prints cold (ms, first request after an API restart) and hot median +(ms, over the hot passes). When several CSVs exist for the same host the newest one wins. +""" +import argparse +import csv +import statistics +from collections import defaultdict +from pathlib import Path + + +def load(paths): + # host -> newest file + newest = {} + for p in paths: + host = p.stem.rsplit('_', 2)[0] + if host not in newest or p.stat().st_mtime > newest[host].stat().st_mtime: + newest[host] = p + data = defaultdict(lambda: defaultdict(lambda: dict(cold=None, cold_each=None, hot=[], status=set()))) + order = [] + for host, p in sorted(newest.items()): + with open(p) as f: + for row in csv.DictReader(f): + d = data[row['name']][host] + ms = float(row['ms']) + if row['phase'] == 'cold': + d['cold'] = ms + elif row['phase'] == 'cold-each': + d['cold_each'] = ms + elif row['phase'] == 'hot': + d['hot'].append(ms) + d['status'].add(row['status']) + if row['name'] not in order: + order.append(row['name']) + return sorted(newest), data, order + + +def fmt(v): + if v is None: + return '-' + return f'{v:,.0f}' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('csvs', nargs='+', type=Path) + ap.add_argument('--sort', choices=['catalog', 'hot', 'cold'], default='catalog', + help='catalog order (default), or by worst hot median / worst cold across hosts.') + ap.add_argument('--top', type=int, help='Only print the first N rows after sorting.') + ap.add_argument('--markdown', action='store_true') + args = ap.parse_args() + + hosts, data, order = load(args.csvs) + + def hot_med(name, host): + h = data[name][host]['hot'] + return statistics.median(h) if h else None + + def worst(name, key): + vals = [data[name][h][key] if key == 'cold' else hot_med(name, h) for h in hosts] + vals = [v for v in vals if v is not None] + return max(vals) if vals else -1 + + names = list(order) + if args.sort == 'hot': + names.sort(key=lambda n: -worst(n, 'hot')) + elif args.sort == 'cold': + names.sort(key=lambda n: -worst(n, 'cold')) + if args.top: + names = names[:args.top] + + cols = ['endpoint'] + [f'{h} cold' for h in hosts] + [f'{h} hot' for h in hosts] + rows = [] + for n in names: + row = [n] + for h in hosts: + d = data[n][h] + c = d['cold_each'] if d['cold_each'] is not None else d['cold'] + bad = d['status'] and d['status'] != {'200'} + row.append(fmt(c) + ('*' if d['cold_each'] is not None else '') + (' !' if bad else '')) + for h in hosts: + row.append(fmt(hot_med(n, h))) + rows.append(row) + + if args.markdown: + print('| ' + ' | '.join(cols) + ' |') + print('|' + '|'.join(['---'] + ['---:'] * (len(cols) - 1)) + '|') + for r in rows: + print('| ' + ' | '.join(r) + ' |') + else: + widths = [max(len(str(x)) for x in col) for col in zip(cols, *rows)] + line = ' '.join(c.ljust(w) if i == 0 else c.rjust(w) for i, (c, w) in enumerate(zip(cols, widths))) + print(line) + print('-' * len(line)) + for r in rows: + print(' '.join(c.ljust(w) if i == 0 else c.rjust(w) for i, (c, w) in enumerate(zip(r, widths)))) + print('\nms; cold = first request after API restart (* = own restart via --cold-each); hot = median of hot passes;' + ' ! = non-200 seen') + + +if __name__ == '__main__': + main() diff --git a/scripts/api_perf_hosts.example.json b/scripts/api_perf_hosts.example.json new file mode 100644 index 00000000..81b70328 --- /dev/null +++ b/scripts/api_perf_hosts.example.json @@ -0,0 +1,29 @@ +{ + "docker": { + "url": "https://192.0.2.10:8443", + "ssh": "user@192.0.2.10", + "ssh_opts": [], + "restart": "docker restart wrolpi-api-1", + "drop_caches": "sudo -n sh -c \"sync; echo 3 > /proc/sys/vm/drop_caches\"" + }, + "pi": { + "url": "https://192.0.2.11", + "ssh": "user@192.0.2.11", + "ssh_opts": [ + "-o", + "IdentitiesOnly=yes" + ], + "restart": "sudo -n systemctl restart wrolpi-api", + "drop_caches": "sudo -n sh -c \"sync; echo 3 > /proc/sys/vm/drop_caches\"" + }, + "pi-limited-sudo": { + "url": "https://192.0.2.12", + "ssh": "wrolpi@192.0.2.12", + "ssh_opts": [ + "-o", + "IdentitiesOnly=yes" + ], + "restart": "sudo -n systemctl stop wrolpi-api.service && sudo -n systemctl start wrolpi-api.service", + "drop_caches": null + } +} diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 2ae851db..ea7654cd 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -1,3 +1,4 @@ +import asyncio import pathlib from http import HTTPStatus @@ -163,9 +164,10 @@ async def worker_status(request: Request): @validate(schema.FilesSearchRequest) async def post_search_files(_: Request, body: schema.FilesSearchRequest): with timer('Searching all files', 'info', logger__=logger): - file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model, - body.tag_names, body.headline, body.months, body.from_year, body.to_year, - body.any_tag, body.order, body.url, body.suffix, body.path, body.deep) + file_groups, total = await asyncio.to_thread( + lib.search_files, body.search_str, body.limit, body.offset, body.mimetypes, body.model, + body.tag_names, body.headline, body.months, body.from_year, body.to_year, + body.any_tag, body.order, body.url, body.suffix, body.path, body.deep) return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total))) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 7a2a6983..ba0d8197 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1149,10 +1149,13 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] return results, total -def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List[dict], int]: +def handle_file_group_search_results(statement: str, params: dict, total: int = None) -> Tuple[List[dict], int]: """ Execute the provided SQL statement and fetch the Files returned. + The statement must select `id` and, unless `total` is provided by the caller, a `total` column + (typically `COUNT(*) OVER()`). + WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search. See: `search` @@ -1160,7 +1163,8 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List with get_db_curs() as curs: curs.execute(statement, params) results = [dict(i) for i in curs.fetchall()] - total = results[0]['total'] if results else 0 + if total is None: + total = results[0]['total'] if results else 0 ordered_ids = [i['id'] for i in results] extras = [dict( ts_rank=i.get('ts_rank'), diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index c99ae954..0e76f5e8 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -1,3 +1,4 @@ +import asyncio import json import pathlib import re @@ -597,8 +598,9 @@ async def get_status(request: Request): description='Get summary statistics of all files', ) async def get_statistics(_): - file_statistics = get_file_statistics() - global_statistics = get_global_statistics() + # Full-table aggregates take many seconds cold on a large library; keep them off the event loop. + file_statistics = await asyncio.to_thread(get_file_statistics) + global_statistics = await asyncio.to_thread(get_global_statistics) return json_response({ 'file_statistics': file_statistics, 'global_statistics': global_statistics, diff --git a/wrolpi/switches.py b/wrolpi/switches.py index f8b9dabb..175fcdc6 100644 --- a/wrolpi/switches.py +++ b/wrolpi/switches.py @@ -96,16 +96,14 @@ async def switch_worker(): # Wait for the switches to be changed. switches_changed: multiprocessing.Event = api_app.shared_ctx.switches_changed + # Must not block the event loop; `perpetual_signal` retries every 0.1s. + switches: dict = api_app.shared_ctx.switches + if not switches_changed.is_set() and not switches: + return + switch_name = None try: # Get new switch and its context. Handle each switch one at a time. - # During testing, don't block the event loop - check for pending switches directly. - # In production, use blocking wait which is more efficient for long-running processes. - if PYTEST and not api_app.shared_ctx.switches: - return - else: - switches_changed.wait(timeout=1) - switches: dict = api_app.shared_ctx.switches with api_app.shared_ctx.switches_lock: switch_name, context = switches.popitem() # Call handler with the stored context, await coroutine, if any. diff --git a/wrolpi/test/test_switches.py b/wrolpi/test/test_switches.py index 2c0f7e5e..af18932b 100644 --- a/wrolpi/test/test_switches.py +++ b/wrolpi/test/test_switches.py @@ -1,5 +1,7 @@ import asyncio import multiprocessing +import time +from unittest import mock import pytest @@ -75,3 +77,16 @@ async def test_invalid_activate_switch(await_switches): with pytest.raises(RuntimeError): switches.activate_switch('test1', 'bad context') + + +@pytest.mark.asyncio +async def test_switch_worker_does_not_block_event_loop(await_switches): + """With no switch pending (the normal state) `switch_worker` must return at once. + + It runs on the Sanic worker's event loop every 0.1s; a blocking `Event.wait(timeout=1)` there + froze every request on that worker for up to a second, ~90% of the time.""" + with mock.patch('wrolpi.switches.PYTEST', False): + start = time.perf_counter() + await switches.switch_worker() + elapsed = time.perf_counter() - start + assert elapsed < 0.5, f'switch_worker blocked for {elapsed:.2f}s with no switch pending'