mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
search: don't COUNT(*) OVER() the page query (#702)
COUNT(*) OVER() forced SQLite to visit every matching file_group before LIMIT, which is a full table scan of the (large) file_group table — ~22s cold on a Pi for videos ordered by download date. Count separately and cache for 60s by filters so indexed orders can stop after one page. Past-the-end pages now report the true total. Videos/archives/docs pagers keep totalPages across page changes.
This commit is contained in:
parent
0b61d3ab08
commit
8008cd1fd8
9 changed files with 436 additions and 74 deletions
|
|
@ -523,7 +523,6 @@ export const useSearchArchives = (defaultLimit) => {
|
|||
|
||||
const localSearchArchives = async () => {
|
||||
setArchives(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline,
|
||||
deep);
|
||||
|
|
@ -594,7 +593,6 @@ export const useSearchDocs = (defaultLimit) => {
|
|||
|
||||
const localSearchDocs = async () => {
|
||||
setDocs(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
const mimetype = docFilterToMimetype(filter);
|
||||
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject,
|
||||
|
|
@ -657,7 +655,6 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
|
|||
const localSearchVideos = async () => {
|
||||
setVideos(null);
|
||||
setLoading(true);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline,
|
||||
censored, deep);
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@ from wrolpi.dates import now, Seconds
|
|||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.lib import handle_file_group_search_results
|
||||
from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \
|
||||
search_filter_cache_key, count_file_groups
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
||||
from wrolpi.tags import tag_append_sub_select_where
|
||||
|
|
@ -1407,9 +1408,8 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
# which case we behave as if no search was requested.
|
||||
file_group_search = fts.file_group_search(search_str, deep=deep) if search_str else None
|
||||
if file_group_search:
|
||||
# A search_str was provided by the user, modify the query to filter by it. The rank and
|
||||
# snippets come from a subquery because FTS5 auxiliary functions (snippet) cannot appear
|
||||
# in a query that also uses window functions (COUNT(*) OVER()).
|
||||
# A search_str was provided by the user, modify the query to filter by it. Rank and
|
||||
# snippets come from a subquery (FTS5 auxiliary functions cannot mix with some SELECT shapes).
|
||||
headline_selects = fts.file_group_headline_selects() if headline else ''
|
||||
fts_join = f'''JOIN (
|
||||
SELECT fts.rowid AS id, {file_group_search.rank_select}{headline_selects}
|
||||
|
|
@ -1425,14 +1425,22 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
fts.b_headline, fts.c_headline, fts.d_headline,
|
||||
fg.title AS title_headline'''
|
||||
|
||||
null_filter = None
|
||||
if order:
|
||||
try:
|
||||
order_by = ARCHIVE_ORDERS[order]
|
||||
except KeyError:
|
||||
raise InvalidOrderBy(f'Invalid order byy {order}')
|
||||
if order in NO_NULL_ORDERS:
|
||||
wheres.append(NO_NULL_ORDERS[order])
|
||||
null_filter = NO_NULL_ORDERS[order]
|
||||
wheres.append(null_filter)
|
||||
|
||||
if not file_group_search and order in ('rank', '-rank'):
|
||||
# Rank without a search used to ORDER BY the windowed total (a constant).
|
||||
order_by = ARCHIVE_ORDERS['-published_datetime'] if order == 'rank' \
|
||||
else ARCHIVE_ORDERS['published_datetime']
|
||||
|
||||
tag_names = tag_names or []
|
||||
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
|
||||
|
||||
if domain:
|
||||
|
|
@ -1442,8 +1450,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
"a.collection_id = (select id from collection where collection.name = :domain and collection.kind = 'domain' LIMIT 1)")
|
||||
|
||||
select_columns = f"{select_columns}," if select_columns else ""
|
||||
wheres = '\n AND '.join(wheres)
|
||||
where = f'WHERE\n{wheres}' if wheres else ''
|
||||
where = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
|
||||
if where or fts_join:
|
||||
from_clause = f'''FROM archive a
|
||||
LEFT JOIN file_group fg ON fg.id = a.file_group_id
|
||||
|
|
@ -1459,7 +1466,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
SELECT
|
||||
a.file_group_id AS id, -- always get `file_group.id` for `handle_file_group_search_results`
|
||||
{select_columns}
|
||||
COUNT(*) OVER() AS total
|
||||
fg.id AS _fg_id
|
||||
{headline_columns}
|
||||
{from_clause}
|
||||
{where}
|
||||
|
|
@ -1468,7 +1475,33 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
'''.strip()
|
||||
logger.debug(f'{stmt} {params}')
|
||||
|
||||
results, total = handle_file_group_search_results(stmt, params)
|
||||
unfiltered = not search_str and not tag_names and not domain and not null_filter
|
||||
if unfiltered:
|
||||
total = cached_search_total(
|
||||
search_filter_cache_key('archives'),
|
||||
lambda: count_file_groups('SELECT COUNT(*) AS total FROM archive', dict()),
|
||||
)
|
||||
else:
|
||||
if file_group_search:
|
||||
count_from = f'''FROM archive a
|
||||
LEFT JOIN file_group fg ON fg.id = a.file_group_id
|
||||
{file_group_search.join}'''
|
||||
count_wheres = list(wheres)
|
||||
count_wheres.append(file_group_search.where)
|
||||
count_where = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
|
||||
count_stmt = f'SELECT COUNT(*) AS total {count_from} {count_where}'
|
||||
else:
|
||||
# Same filters as the page, but do not walk the date index just to count.
|
||||
count_from = '''FROM archive a
|
||||
LEFT JOIN file_group fg ON fg.id = a.file_group_id'''
|
||||
count_stmt = f'SELECT COUNT(*) AS total {count_from} {where}'
|
||||
cache_key = search_filter_cache_key(
|
||||
'archives', search_str=search_str, domain=domain, tag_names=tag_names,
|
||||
deep=deep, null_filter=null_filter,
|
||||
)
|
||||
total = cached_search_total(cache_key, lambda: count_file_groups(count_stmt, params))
|
||||
|
||||
results, total = handle_file_group_search_results(stmt, params, total=total)
|
||||
|
||||
if file_group_search and headline and results:
|
||||
# Highlight the plain titles like the FTS snippets above (Postgres used ts_headline).
|
||||
|
|
|
|||
|
|
@ -158,6 +158,10 @@ async def test_search_offset(test_session, archive_factory, async_client):
|
|||
await check_results(async_client, data, list(range(300, 280, -1)))
|
||||
data = {'search_str': None, 'offset': 500}
|
||||
await check_results(async_client, data, [])
|
||||
# Past the last page still reports the true total (a separate COUNT, not COUNT(*) OVER()).
|
||||
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['totals']['file_groups'] == 500
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ from wrolpi.collections.models import Collection
|
|||
from wrolpi.collections.types import collection_type_registry
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype, cached_search_total, \
|
||||
search_filter_cache_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -388,7 +389,19 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
|
|||
.subquery()
|
||||
query = query.filter(FileGroup.id.in_(tagged_fg_ids))
|
||||
|
||||
total = query.count()
|
||||
unfiltered = not (search_str or author or subject or language or mimetype or tag_names)
|
||||
if unfiltered:
|
||||
total = cached_search_total(
|
||||
search_filter_cache_key('docs'),
|
||||
lambda: session.query(func.count(Doc.id)).scalar() or 0,
|
||||
)
|
||||
else:
|
||||
cache_key = search_filter_cache_key(
|
||||
'docs', search_str=search_str, author=author, subject=subject,
|
||||
language=language, mimetype=mimetype, tag_names=tag_names, deep=deep,
|
||||
)
|
||||
# Count before ORDER BY / LIMIT; cache by filters not page.
|
||||
total = cached_search_total(cache_key, lambda q=query: q.count())
|
||||
|
||||
# Ordering.
|
||||
if order_by == 'rank' and fts_sq is not None:
|
||||
|
|
|
|||
|
|
@ -669,8 +669,7 @@ async def test_search_videos_default_page_pagination(test_session, video_factory
|
|||
await assert_video_search(order_by='-published_datetime', limit=2, offset=0, assert_total=6, assert_ids=newest[0:2])
|
||||
await assert_video_search(order_by='-published_datetime', limit=2, offset=2, assert_total=6, assert_ids=newest[2:4])
|
||||
await assert_video_search(order_by='-published_datetime', limit=2, offset=4, assert_total=6, assert_ids=newest[4:6])
|
||||
# Past the end the fast path still reports the true total (the generic query has no rows to
|
||||
# carry `COUNT(*) OVER()` in, so it reports 0 there; the fast path is the better behavior).
|
||||
# Past the end both the date fast path and the generic query report the true total.
|
||||
await assert_video_search(order_by='-published_datetime', limit=2, offset=6, assert_total=6, assert_ids=[])
|
||||
|
||||
# Oldest first: NULL dates first (SQLite sorts NULL lowest), ties broken by id ascending.
|
||||
|
|
@ -691,32 +690,38 @@ async def test_search_videos_default_page_pagination(test_session, video_factory
|
|||
generic = ids(order, limit, offset)
|
||||
assert fast == generic, f'{order} limit={limit} offset={offset}: fast {fast} != generic {generic}'
|
||||
|
||||
# Generic orders used to report total=0 past the last page because COUNT(*) OVER() had no
|
||||
# rows to ride on. A separate COUNT reports the true total, like the date fast path.
|
||||
await assert_video_search(order_by='-download_datetime', limit=2, offset=6, assert_total=6, assert_ids=[])
|
||||
await assert_video_search(order_by='-size', limit=2, offset=6, assert_total=6, assert_ids=[])
|
||||
|
||||
|
||||
def test_cached_total(test_session):
|
||||
"""The unfiltered Videos page total is cached briefly: counting every video is a full index scan
|
||||
that costs seconds cold on a large library, and pagination can tolerate a slightly stale total."""
|
||||
from modules.videos.video import lib
|
||||
"""Search totals are cached briefly: counting a large library is a full index scan
|
||||
that costs seconds cold, and pagination can tolerate a slightly stale total."""
|
||||
from wrolpi.files import lib as files_lib
|
||||
calls = []
|
||||
|
||||
def compute():
|
||||
calls.append(1)
|
||||
return 42
|
||||
|
||||
lib._TOTALS_CACHE.clear()
|
||||
with mock.patch('modules.videos.video.lib.PYTEST', False):
|
||||
assert lib._cached_total('k', compute) == 42
|
||||
assert lib._cached_total('k', compute) == 42
|
||||
files_lib._SEARCH_TOTALS_CACHE.clear()
|
||||
with mock.patch('wrolpi.files.lib.PYTEST', False):
|
||||
assert files_lib.cached_search_total('k', compute) == 42
|
||||
assert files_lib.cached_search_total('k', compute) == 42
|
||||
assert len(calls) == 1, 'second call within the TTL should hit the cache'
|
||||
# Expire the entry.
|
||||
cached_at, total = lib._TOTALS_CACHE['k']
|
||||
lib._TOTALS_CACHE['k'] = (cached_at - lib.TOTALS_CACHE_SECONDS - 1, total)
|
||||
assert lib._cached_total('k', compute) == 42
|
||||
cached_at, total = files_lib._SEARCH_TOTALS_CACHE['k']
|
||||
files_lib._SEARCH_TOTALS_CACHE['k'] = (
|
||||
cached_at - files_lib.SEARCH_TOTALS_CACHE_SECONDS - 1, total)
|
||||
assert files_lib.cached_search_total('k', compute) == 42
|
||||
assert len(calls) == 2, 'expired entry should be recomputed'
|
||||
|
||||
# Tests need exact totals, so the cache is bypassed under pytest.
|
||||
assert lib._cached_total('k', compute) == 42
|
||||
assert len(calls) == 3
|
||||
lib._TOTALS_CACHE.clear()
|
||||
assert files_lib.cached_search_total('k', compute) == 42
|
||||
assert len(calls) == 3, 'pytest should bypass the cache'
|
||||
|
||||
|
||||
async def test_video_description_endpoint(async_client, test_session, video_factory):
|
||||
"""The description is served by its own endpoint, like comments and captions. It comes from the
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
import asyncio
|
||||
import random
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from typing import Tuple, Optional, List, Dict, Callable
|
||||
from typing import Tuple, Optional, List
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import or_
|
||||
|
|
@ -14,10 +13,11 @@ from wrolpi.common import logger, limit_concurrent, wrol_mode_check
|
|||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.files.lib import handle_file_group_search_results
|
||||
from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \
|
||||
search_filter_cache_key, count_file_groups
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import tag_append_sub_select_where
|
||||
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR, PYTEST
|
||||
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR
|
||||
from ..cookies import cookies_unlocked, cookies_for_download
|
||||
from ..lib import get_yt_dlp_http_headers, get_yt_dlp_sleep_opts
|
||||
|
||||
|
|
@ -91,22 +91,6 @@ VIDEO_QUERY_LIMIT = 24
|
|||
INDEXED_DATE_ORDERS = ('published_datetime', '-published_datetime')
|
||||
VIDEO_MIMETYPE_PREFIXES = ('video/', 'audio/')
|
||||
|
||||
# The unfiltered Videos page total, cached for a short time. Counting every video walks the whole
|
||||
# mimetype index, which costs seconds cold on a large library on a spinning disk; pagination can
|
||||
# tolerate a total that is a minute stale. Bypassed under pytest so tests see exact totals.
|
||||
TOTALS_CACHE_SECONDS = 60
|
||||
_TOTALS_CACHE: Dict[str, Tuple[float, int]] = dict()
|
||||
|
||||
|
||||
def _cached_total(key: str, compute: Callable[[], int]) -> int:
|
||||
if not PYTEST:
|
||||
hit = _TOTALS_CACHE.get(key)
|
||||
if hit and time.monotonic() - hit[0] < TOTALS_CACHE_SECONDS:
|
||||
return hit[1]
|
||||
total = compute()
|
||||
_TOTALS_CACHE[key] = (time.monotonic(), total)
|
||||
return total
|
||||
|
||||
|
||||
def _video_mimetypes() -> List[str]:
|
||||
"""Every distinct video/* and audio/* mimetype in the library (a handful; served by the mimetype index)."""
|
||||
|
|
@ -133,12 +117,11 @@ def _count_videos() -> int:
|
|||
def _search_videos_by_date(order: str, limit: int, offset: int) -> Tuple[List[dict], int]:
|
||||
"""The unfiltered Videos page, ordered by date, without visiting every video.
|
||||
|
||||
The generic query (`mimetype LIKE 'video/%' OR ...` + `COUNT(*) OVER()`) must visit every
|
||||
video's index entry before it can apply LIMIT, which scales with the library rather than the
|
||||
page. Here each exact mimetype is a pure covering-index seek on (mimetype, effective_datetime)
|
||||
that reads only `limit + offset` entries; those few rows are then merged. Results must be
|
||||
identical to the generic query's (same ORDER BY, including the id tiebreaker and NULL placement).
|
||||
The total is a separate, cached count."""
|
||||
The generic query (`mimetype LIKE 'video/%' OR ...`) plus a windowed COUNT used to visit every
|
||||
video before LIMIT. Here each exact mimetype is a covering-index seek on
|
||||
(mimetype, effective_datetime) that reads only `limit + offset` entries; those few rows are
|
||||
then merged. Results must be identical to the generic query's (same ORDER BY, including the
|
||||
id tiebreaker and NULL placement). The total is a separate, cached count."""
|
||||
order_by = VIDEO_ORDERS[order]
|
||||
mimetypes = _video_mimetypes()
|
||||
if not mimetypes:
|
||||
|
|
@ -160,7 +143,7 @@ def _search_videos_by_date(order: str, limit: int, offset: int) -> Tuple[List[di
|
|||
'''.strip()
|
||||
logger.debug(f'{stmt} {params}')
|
||||
|
||||
total = _cached_total('videos', _count_videos)
|
||||
total = cached_search_total(search_filter_cache_key('videos'), _count_videos)
|
||||
return handle_file_group_search_results(stmt, params, total=total)
|
||||
|
||||
|
||||
|
|
@ -199,17 +182,15 @@ def search_videos(
|
|||
params['channel_id'] = channel_id
|
||||
|
||||
# `deep` searches d_text (captions) as well; the default abc search is much smaller and faster.
|
||||
# The subquery join is required: snippet() cannot coexist with COUNT(*) OVER() in one SELECT.
|
||||
# Rank/snippets come from a subquery so they stay valid if a caller adds window functions.
|
||||
fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \
|
||||
if search_str else None
|
||||
if fts_search:
|
||||
# A search_str was provided by the user, modify the query to filter by it.
|
||||
select_columns = f'fg.id, {fts_search.rank_select}, COUNT(*) OVER() AS total'
|
||||
select_columns = f'fg.id, {fts_search.rank_select}'
|
||||
joins.append(fts_search.join)
|
||||
params.update(fts_search.params)
|
||||
else:
|
||||
# No (usable) search_str provided. Get id and total only.
|
||||
select_columns = 'fg.id, COUNT(*) OVER() AS total'
|
||||
select_columns = 'fg.id'
|
||||
|
||||
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
|
||||
|
||||
|
|
@ -222,21 +203,27 @@ def search_videos(
|
|||
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
|
||||
# whitelist.
|
||||
order_by = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
|
||||
null_filter = None
|
||||
if order:
|
||||
try:
|
||||
order_by = VIDEO_ORDERS[order]
|
||||
except KeyError:
|
||||
raise
|
||||
if order in NO_NULL_ORDERS:
|
||||
wheres.append(NO_NULL_ORDERS[order])
|
||||
null_filter = NO_NULL_ORDERS[order]
|
||||
wheres.append(null_filter)
|
||||
if order in JOIN_ORDERS:
|
||||
join_video = True
|
||||
|
||||
effective_order = order or DEFAULT_VIDEO_ORDER
|
||||
if not fts_search and effective_order in ('rank', '-rank'):
|
||||
# Rank without a search used to ORDER BY the windowed total (a constant). Equivalent: id.
|
||||
order_by = 'fg.id ASC' if effective_order == '-rank' else 'fg.id DESC'
|
||||
|
||||
if join_video:
|
||||
joins.insert(0, 'LEFT JOIN video v on v.file_group_id = fg.id')
|
||||
|
||||
wheres = '\n AND '.join(wheres)
|
||||
where = f'WHERE\n{wheres}' if wheres else ''
|
||||
where = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
|
||||
join = '\n'.join(joins)
|
||||
stmt = f'''
|
||||
SELECT
|
||||
|
|
@ -250,7 +237,35 @@ def search_videos(
|
|||
'''.strip()
|
||||
logger.debug(f'{stmt} {params}')
|
||||
|
||||
results, total = handle_file_group_search_results(stmt, params)
|
||||
unfiltered = (not search_str and not tag_names and not channel_id and not censored
|
||||
and not null_filter)
|
||||
if unfiltered:
|
||||
total = cached_search_total(search_filter_cache_key('videos'), _count_videos)
|
||||
else:
|
||||
if fts_search:
|
||||
fts_count = fts.file_group_search(search_str, deep=deep)
|
||||
count_joins = [j for j in joins if 'file_group_fts' not in j]
|
||||
if fts_count:
|
||||
count_joins.append(fts_count.join)
|
||||
count_wheres = [w for w in wheres]
|
||||
if fts_count:
|
||||
count_wheres.append(fts_count.where)
|
||||
count_join = '\n'.join(count_joins)
|
||||
count_where = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
|
||||
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {count_join} {count_where}'
|
||||
count_params = dict(params)
|
||||
if fts_count:
|
||||
count_params.update(fts_count.params)
|
||||
else:
|
||||
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where}'
|
||||
count_params = params
|
||||
cache_key = search_filter_cache_key(
|
||||
'videos', search_str=search_str, channel_id=channel_id, tag_names=tag_names,
|
||||
censored=censored, deep=deep, null_filter=null_filter,
|
||||
)
|
||||
total = cached_search_total(cache_key, lambda: count_file_groups(count_stmt, count_params))
|
||||
|
||||
results, total = handle_file_group_search_results(stmt, params, total=total)
|
||||
return results, total
|
||||
|
||||
|
||||
|
|
|
|||
209
scripts/search_order_perf.py
Normal file
209
scripts/search_order_perf.py
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Benchmark file-search endpoints across every supported order.
|
||||
|
||||
Usage:
|
||||
python3 scripts/search_order_perf.py --host decay --label before
|
||||
python3 scripts/search_order_perf.py --host decay --label after --passes 3
|
||||
|
||||
Writes CSV to ./perf_results/search_order_<host>_<label>_<timestamp>.csv and prints a table.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
import ssl
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
HOSTS_FILE = Path(__file__).with_name('api_perf_hosts.json')
|
||||
|
||||
SSL_CTX = ssl.create_default_context()
|
||||
SSL_CTX.check_hostname = False
|
||||
SSL_CTX.verify_mode = ssl.CERT_NONE
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
OPENER = urllib.request.build_opener(_NoRedirect, urllib.request.HTTPSHandler(context=SSL_CTX))
|
||||
|
||||
|
||||
def load_hosts() -> dict:
|
||||
if not HOSTS_FILE.exists():
|
||||
return {}
|
||||
return json.loads(HOSTS_FILE.read_text())
|
||||
|
||||
|
||||
# (name, path, base_body, orders)
|
||||
# orders: None means a single request with whatever base_body already has.
|
||||
CASES = [
|
||||
('videos', '/api/videos/search',
|
||||
{'offset': 0, 'limit': 24, 'tag_names': []},
|
||||
[
|
||||
'published_datetime', '-published_datetime',
|
||||
'download_datetime', '-download_datetime',
|
||||
'size', '-size',
|
||||
'length', '-length',
|
||||
'size_to_duration', '-size_to_duration',
|
||||
'viewed', '-viewed',
|
||||
'view_count', '-view_count',
|
||||
]),
|
||||
('videos/search', '/api/videos/search',
|
||||
{'offset': 0, 'limit': 24, 'tag_names': [], 'search_str': 'the'},
|
||||
['rank', '-rank', '-published_datetime', '-download_datetime', '-size']),
|
||||
('archive', '/api/archive/search',
|
||||
{'offset': 0, 'limit': 20},
|
||||
[
|
||||
'published_datetime', '-published_datetime',
|
||||
'download_datetime', '-download_datetime',
|
||||
'published_modified_datetime', '-published_modified_datetime',
|
||||
'size', '-size',
|
||||
'viewed', '-viewed',
|
||||
]),
|
||||
('archive/search', '/api/archive/search',
|
||||
{'offset': 0, 'limit': 20, 'search_str': 'the'},
|
||||
['rank', '-rank', '-published_datetime', '-download_datetime', '-size']),
|
||||
('files', '/api/files/search',
|
||||
{'offset': 0, 'limit': 20},
|
||||
[None, 'viewed', '-viewed']),
|
||||
('files/search', '/api/files/search',
|
||||
{'offset': 0, 'limit': 20, 'search_str': 'the'},
|
||||
[None, 'viewed', '-viewed']),
|
||||
('docs', '/api/docs/search',
|
||||
{'limit': 20, 'offset': 0},
|
||||
['published_datetime', 'size', 'title']),
|
||||
('docs/search', '/api/docs/search',
|
||||
{'limit': 20, 'offset': 0, 'search_str': 'the'},
|
||||
['rank', 'published_datetime', 'size', 'title']),
|
||||
]
|
||||
|
||||
|
||||
def request(base_url: str, path: str, body: dict, timeout: float = 180.0) -> dict:
|
||||
url = base_url.rstrip('/') + path
|
||||
data = json.dumps(body).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method='POST',
|
||||
headers={'Accept': 'application/json', 'Content-Type': 'application/json'},
|
||||
)
|
||||
start = time.perf_counter()
|
||||
status, size, error, total = None, 0, None, None
|
||||
try:
|
||||
with OPENER.open(req, timeout=timeout) as resp:
|
||||
payload = resp.read()
|
||||
status = resp.status
|
||||
size = len(payload)
|
||||
if status == 200:
|
||||
try:
|
||||
total = json.loads(payload).get('totals', {}).get('file_groups')
|
||||
except Exception:
|
||||
pass
|
||||
except urllib.error.HTTPError as e:
|
||||
status = e.code
|
||||
error = f'HTTP {e.code}'
|
||||
try:
|
||||
size = len(e.read())
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
error = type(e).__name__ + (f': {e}' if str(e) else '')
|
||||
elapsed_ms = round((time.perf_counter() - start) * 1000, 1)
|
||||
return dict(ms=elapsed_ms, status=status, bytes=size, error=error, total=total)
|
||||
|
||||
|
||||
def expand_cases():
|
||||
rows = []
|
||||
for group, path, base, orders in CASES:
|
||||
for order in orders:
|
||||
body = dict(base)
|
||||
label = group
|
||||
if order is not None:
|
||||
if path == '/api/files/search':
|
||||
body['order'] = order
|
||||
else:
|
||||
body['order_by'] = order
|
||||
label = f'{group} order={order}'
|
||||
else:
|
||||
label = f'{group} order=<default>'
|
||||
rows.append((label, path, body))
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
hosts = load_hosts()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--host', default='decay', choices=sorted(hosts) or ['decay'])
|
||||
ap.add_argument('--url', default=None, help='Override host URL')
|
||||
ap.add_argument('--label', default='run', help='Tag written into the CSV filename (before/after)')
|
||||
ap.add_argument('--passes', type=int, default=3)
|
||||
ap.add_argument('--warmup', type=int, default=1)
|
||||
ap.add_argument('--out-dir', default='perf_results')
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.url:
|
||||
base_url = args.url
|
||||
host_name = 'adhoc'
|
||||
else:
|
||||
profile = hosts[args.host]
|
||||
base_url = profile['url']
|
||||
host_name = args.host
|
||||
|
||||
cases = expand_cases()
|
||||
print(f'host={host_name} url={base_url} label={args.label} cases={len(cases)} passes={args.passes}')
|
||||
|
||||
if args.warmup:
|
||||
print(f'warmup x{args.warmup}...')
|
||||
for _ in range(args.warmup):
|
||||
for _, path, body in cases:
|
||||
request(base_url, path, body)
|
||||
|
||||
results = []
|
||||
for label, path, body in cases:
|
||||
times = []
|
||||
last = None
|
||||
for i in range(args.passes):
|
||||
last = request(base_url, path, body)
|
||||
times.append(last['ms'])
|
||||
status = last['status']
|
||||
err = f' {last["error"]}' if last['error'] else ''
|
||||
print(f' {last["ms"]:8.1f} ms {status} {label}{err}')
|
||||
med = statistics.median(times)
|
||||
results.append(dict(
|
||||
label=label,
|
||||
path=path,
|
||||
order=body.get('order_by') or body.get('order') or '',
|
||||
search=bool(body.get('search_str')),
|
||||
median_ms=med,
|
||||
min_ms=min(times),
|
||||
max_ms=max(times),
|
||||
status=last['status'] if last else None,
|
||||
total=last['total'] if last else None,
|
||||
bytes=last['bytes'] if last else None,
|
||||
error=last['error'] if last else None,
|
||||
))
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(exist_ok=True)
|
||||
stamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
out = out_dir / f'search_order_{host_name}_{args.label}_{stamp}.csv'
|
||||
with out.open('w', newline='') as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(results[0].keys()))
|
||||
w.writeheader()
|
||||
w.writerows(results)
|
||||
|
||||
print()
|
||||
print(f'{"median_ms":>10} {"min":>8} {"max":>8} {"total":>8} case')
|
||||
print('-' * 80)
|
||||
for r in results:
|
||||
total = '' if r['total'] is None else str(r['total'])
|
||||
print(f'{r["median_ms"]:10.1f} {r["min_ms"]:8.1f} {r["max_ms"]:8.1f} {total:>8} {r["label"]}')
|
||||
print(f'\nwrote {out}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main() or 0)
|
||||
|
|
@ -11,6 +11,7 @@ import re
|
|||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from itertools import zip_longest
|
||||
|
|
@ -56,7 +57,8 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix',
|
|||
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
|
||||
'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href',
|
||||
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
|
||||
'get_normalized_ignored_directories', 'sanitize_ignored_directories']
|
||||
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
|
||||
'cached_search_total', 'search_filter_cache_key', 'count_file_groups']
|
||||
|
||||
|
||||
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
|
||||
|
|
@ -1038,6 +1040,45 @@ async def search_directories_by_name(session: Session, name: str, excluded: List
|
|||
return directories
|
||||
|
||||
|
||||
# Pagination totals: counting a large library is a full index walk that costs seconds cold on a
|
||||
# spinning disk. Pagination can tolerate a total that is a minute stale. Bypassed under pytest
|
||||
# so tests see exact totals. Keyed by filters, never by offset/limit/order (except filters that
|
||||
# an order implies, e.g. viewed IS NOT NULL).
|
||||
SEARCH_TOTALS_CACHE_SECONDS = 60
|
||||
_SEARCH_TOTALS_CACHE: Dict[str, Tuple[float, int]] = dict()
|
||||
|
||||
|
||||
def cached_search_total(key: str, compute: Callable[[], int]) -> int:
|
||||
"""Return `compute()`; cache the result briefly unless running under pytest."""
|
||||
if not PYTEST:
|
||||
hit = _SEARCH_TOTALS_CACHE.get(key)
|
||||
if hit and time.monotonic() - hit[0] < SEARCH_TOTALS_CACHE_SECONDS:
|
||||
return hit[1]
|
||||
total = compute()
|
||||
_SEARCH_TOTALS_CACHE[key] = (time.monotonic(), total)
|
||||
return total
|
||||
|
||||
|
||||
def search_filter_cache_key(corpus: str, **filters) -> str:
|
||||
"""Stable cache key for a search total. Omits empty/false filters."""
|
||||
payload = {'corpus': corpus}
|
||||
for k, v in filters.items():
|
||||
if v in (None, '', [], (), False):
|
||||
continue
|
||||
if isinstance(v, (list, tuple, set)):
|
||||
v = sorted(v)
|
||||
payload[k] = v
|
||||
return json.dumps(payload, sort_keys=True, default=str)
|
||||
|
||||
|
||||
def count_file_groups(statement: str, params: dict) -> int:
|
||||
"""Run a `SELECT COUNT(*) AS total ...` statement and return the integer."""
|
||||
with get_db_curs() as curs:
|
||||
curs.execute(statement, params)
|
||||
row = curs.fetchone()
|
||||
return int(row['total'])
|
||||
|
||||
|
||||
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
|
||||
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
|
||||
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
|
||||
|
|
@ -1127,25 +1168,56 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
elif order == '-viewed':
|
||||
order_by = 'viewed ASC NULLS LAST, 1 ASC'
|
||||
|
||||
if order and not search_str:
|
||||
viewed_only = bool(order and not search_str)
|
||||
if viewed_only:
|
||||
# Only filter out unviewed files if search_str is not provided.
|
||||
wheres.append('viewed IS NOT NULL')
|
||||
|
||||
wheres = '\n AND '.join(wheres)
|
||||
selects = f"{', '.join(selects)}, " if selects else ""
|
||||
where_sql = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
|
||||
extra_selects = f', {", ".join(selects)}' if selects else ''
|
||||
join = '\n'.join(joins)
|
||||
stmt = f'''
|
||||
SELECT fg.id, {selects} COUNT(*) OVER() AS total
|
||||
SELECT fg.id{extra_selects}
|
||||
{headline}
|
||||
FROM file_group fg
|
||||
{join}
|
||||
{f"WHERE {wheres}" if wheres else ""}
|
||||
{where_sql}
|
||||
ORDER BY {order_by}
|
||||
LIMIT :limit OFFSET :offset
|
||||
'''
|
||||
logger.debug(stmt)
|
||||
|
||||
results, total = handle_file_group_search_results(stmt, params)
|
||||
# COUNT(*) OVER() in the page query forces SQLite to visit every match before LIMIT.
|
||||
# Count separately (and cache) so the page can use the datetime/id index.
|
||||
if not search_str and not tag_names and not mimetypes and not model and not months \
|
||||
and not from_year and not to_year and not any_tag and not url and not suffix \
|
||||
and not path and not viewed_only:
|
||||
count_stmt = 'SELECT COUNT(*) AS total FROM file_group'
|
||||
count_params = dict()
|
||||
elif fts_search:
|
||||
# Count via a direct FTS join (no ranking) rather than the snippet subquery.
|
||||
fts_count = fts.file_group_search(search_str, deep=deep)
|
||||
count_wheres = list(wheres)
|
||||
if fts_count:
|
||||
count_wheres.append(fts_count.where)
|
||||
count_where_sql = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
|
||||
count_join = fts_count.join if fts_count else ''
|
||||
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {count_join} {count_where_sql}'
|
||||
count_params = dict(params)
|
||||
if fts_count:
|
||||
count_params.update(fts_count.params)
|
||||
else:
|
||||
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where_sql}'
|
||||
count_params = params
|
||||
|
||||
cache_key = search_filter_cache_key(
|
||||
'files', search_str=search_str, mimetypes=mimetypes, model=model,
|
||||
tag_names=tag_names, months=months, from_year=from_year, to_year=to_year,
|
||||
any_tag=any_tag, url=url, suffix=suffix, path=path, deep=deep,
|
||||
viewed_only=viewed_only,
|
||||
)
|
||||
total = cached_search_total(cache_key, lambda: count_file_groups(count_stmt, count_params))
|
||||
results, total = handle_file_group_search_results(stmt, params, total=total)
|
||||
return results, total
|
||||
|
||||
|
||||
|
|
@ -1153,8 +1225,8 @@ def handle_file_group_search_results(statement: str, params: dict, total: int =
|
|||
"""
|
||||
Execute the provided SQL statement and fetch the Files returned.
|
||||
|
||||
The statement must select `id` and, unless `total` is provided by the caller, a `total` column
|
||||
(typically `COUNT(*) OVER()`).
|
||||
The statement must select `id`. Pass `total` from a separate COUNT (callers should not use
|
||||
`COUNT(*) OVER()` in the page query; it prevents SQLite from applying LIMIT via an index).
|
||||
|
||||
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search.
|
||||
|
||||
|
|
|
|||
|
|
@ -1101,6 +1101,11 @@ async def test_file_search_date_range(async_client, test_session, example_pdf, e
|
|||
files, total = lib.search_files('', 10, 0)
|
||||
assert total == 2, 'Should only be 2 files.'
|
||||
|
||||
# Past the last page still reports the true total (count is a separate query).
|
||||
files, total = lib.search_files('', 10, 10)
|
||||
assert files == []
|
||||
assert total == 2
|
||||
|
||||
# PDF was published in December.
|
||||
files, total = lib.search_files('', 10, 0, months=[12, ])
|
||||
assert total == 1, 'Only PDF should be from December 2022'
|
||||
|
|
@ -1120,6 +1125,15 @@ async def test_file_search_date_range(async_client, test_session, example_pdf, e
|
|||
assert files[0]['id'] == pdf.id
|
||||
|
||||
|
||||
def test_search_filter_cache_key():
|
||||
"""Totals are cached by filters, not by page or sort direction."""
|
||||
a = lib.search_filter_cache_key('videos', deep=False, tag_names=[], search_str='')
|
||||
b = lib.search_filter_cache_key('videos')
|
||||
assert a == b
|
||||
assert lib.search_filter_cache_key('videos', tag_names=['b', 'a']) == \
|
||||
lib.search_filter_cache_key('videos', tag_names=['a', 'b'])
|
||||
|
||||
|
||||
def test_replace_file(test_directory):
|
||||
file = test_directory / 'foo.txt'
|
||||
file.write_text('old contents')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue