rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
wrolpi/modules/videos/video/lib.py
lrnselfreliance 8008cd1fd8
search: don't COUNT(*) OVER() the page query (#702) 
COUNT(*) OVER() forced SQLite to visit every matching file_group
before LIMIT, which is a full table scan of the (large) file_group
table — ~22s cold on a Pi for videos ordered by download date.

Count separately and cache for 60s by filters so indexed orders can
stop after one page.  Past-the-end pages now report the true total.
Videos/archives/docs pagers keep totalPages across page changes.
2026-09-12 14:44:30 -06:00

433 lines
18 KiB
Python
Raw Blame History

import asyncio
import random
from datetime import timedelta
from typing import Tuple, Optional, List
import yt_dlp
from sqlalchemy import or_
from sqlalchemy.orm import Session
from modules.videos.models import Video, Channel
from wrolpi import fts
from wrolpi.common import logger, limit_concurrent, wrol_mode_check
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import download_manager
from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \
search_filter_cache_key, count_file_groups
from wrolpi.files.models import FileGroup
from wrolpi.tags import tag_append_sub_select_where
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR
from ..cookies import cookies_unlocked, cookies_for_download
from ..lib import get_yt_dlp_http_headers, get_yt_dlp_sleep_opts
logger.getChild(__name__)
def get_video_for_app(file_group_id: int, skip_viewed: bool = False) -> Tuple[dict, Optional[dict], Optional[dict]]:
"""
Get a Video by its FileGroup ID, with its prev/next videos. Mark the Video as viewed.
"""
with get_db_session(commit=True) as session:
video = Video.find_by_file_group_id(session, file_group_id)
if not skip_viewed:
video.file_group.set_viewed()
previous_video, next_video = video.get_surrounding_videos()
video = video.__json__()
previous_video = previous_video.__json__() if previous_video and previous_video.file_group else None
next_video = next_video.__json__() if next_video and next_video.file_group else None
return video, previous_video, next_video
def get_video(file_group_id: int) -> Video:
"""
Get a Video by its FileGroup ID.
"""
with get_db_session() as session:
video = Video.find_by_file_group_id(session, file_group_id)
return video
VIDEO_ORDERS = {
# fg.id (the rowid) is the pagination tiebreaker: it is present in every index, so these
# orders stay index-only. LOWER(fg.primary_path) forced SQLite to read every (large) row.
# Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
'published_datetime': 'fg.effective_datetime ASC, fg.id ASC',
'-published_datetime': 'fg.effective_datetime DESC NULLS LAST, fg.id ASC',
'rank': '2 DESC, fg.id DESC',
'-rank': '2 ASC, fg.id ASC',
'size': 'fg.size ASC, fg.id ASC',
'-size': 'fg.size DESC, fg.id DESC',
'length': 'fg.length ASC, fg.id ASC',
'-length': 'fg.length DESC, fg.id DESC',
'size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END ASC, fg.id ASC',
'-size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END DESC, fg.id DESC',
'viewed': 'fg.viewed ASC',
'-viewed': 'fg.viewed DESC',
'view_count': 'v.view_count ASC',
'-view_count': 'v.view_count DESC',
'download_datetime': 'fg.download_datetime ASC, fg.id ASC',
'-download_datetime': 'fg.download_datetime DESC NULLS LAST, fg.id ASC',
}
NO_NULL_ORDERS = {
'viewed': 'fg.viewed IS NOT NULL',
'-viewed': 'fg.viewed IS NOT NULL',
'length': 'fg.length IS NOT NULL',
'-length': 'fg.length IS NOT NULL',
'size': 'fg.size IS NOT NULL',
'-size': 'fg.size IS NOT NULL',
'size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
'-size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
'view_count': 'v.view_count IS NOT NULL',
'-view_count': 'v.view_count IS NOT NULL',
}
JOIN_ORDERS = ('view_count', '-view_count')
DEFAULT_VIDEO_ORDER = 'rank'
VIDEO_QUERY_LIMIT = 24
# Orders the unfiltered Videos page can serve straight from the (mimetype, effective_datetime) index.
INDEXED_DATE_ORDERS = ('published_datetime', '-published_datetime')
VIDEO_MIMETYPE_PREFIXES = ('video/', 'audio/')
def _video_mimetypes() -> List[str]:
"""Every distinct video/* and audio/* mimetype in the library (a handful; served by the mimetype index)."""
mimetypes = []
with get_db_curs() as curs:
for prefix in VIDEO_MIMETYPE_PREFIXES:
# '0' is the character after '/', so this is the half-open range of the prefix.
curs.execute('SELECT DISTINCT mimetype FROM file_group WHERE mimetype >= ? AND mimetype < ?',
(prefix, prefix[:-1] + '0'))
mimetypes.extend(i[0] for i in curs.fetchall())
return mimetypes
def _count_videos() -> int:
ranges = ' OR '.join(f'(mimetype >= :p{i} AND mimetype < :e{i})' for i in range(len(VIDEO_MIMETYPE_PREFIXES)))
params = dict()
for i, prefix in enumerate(VIDEO_MIMETYPE_PREFIXES):
params[f'p{i}'], params[f'e{i}'] = prefix, prefix[:-1] + '0'
with get_db_curs() as curs:
curs.execute(f'SELECT COUNT(*) FROM file_group WHERE {ranges}', params)
return curs.fetchone()[0]
def _search_videos_by_date(order: str, limit: int, offset: int) -> Tuple[List[dict], int]:
"""The unfiltered Videos page, ordered by date, without visiting every video.
The generic query (`mimetype LIKE 'video/%' OR ...`) plus a windowed COUNT used to visit every
video before LIMIT. Here each exact mimetype is a covering-index seek on
(mimetype, effective_datetime) that reads only `limit + offset` entries; those few rows are
then merged. Results must be identical to the generic query's (same ORDER BY, including the
id tiebreaker and NULL placement). The total is a separate, cached count."""
order_by = VIDEO_ORDERS[order]
mimetypes = _video_mimetypes()
if not mimetypes:
return [], 0
n = limit + offset
params = {f'mt{i}': mimetype for i, mimetype in enumerate(mimetypes)}
params['offset'] = offset
members = '\n UNION ALL\n'.join(
f'SELECT * FROM (SELECT fg.id, fg.effective_datetime FROM file_group fg'
f' WHERE fg.mimetype = :mt{i} ORDER BY {order_by} LIMIT {n})'
for i in range(len(mimetypes))
)
stmt = f'''
SELECT fg.id
FROM ({members}) fg
ORDER BY {order_by}
LIMIT {limit} OFFSET :offset
'''.strip()
logger.debug(f'{stmt} {params}')
total = cached_search_total(search_filter_cache_key('videos'), _count_videos)
return handle_file_group_search_results(stmt, params, total=total)
def search_videos(
search_str: str = None,
offset: int = None,
limit: int = VIDEO_QUERY_LIMIT,
channel_id: int = None,
order: str = None,
tag_names: List[str] = None,
headline: bool = False,
censored: bool = False,
deep: bool = False,
) -> Tuple[List[dict], int]:
tag_names = tag_names or []
# Search videos and audio-only files.
wheres = ["(fg.mimetype LIKE 'video/%' OR fg.mimetype LIKE 'audio/%')"]
if censored:
# Only return videos which are no longer available for download.
wheres.append('fg.censored = TRUE')
joins = list()
join_video = False
# Explicit JSON nulls bypass the schema defaults; SQLite rejects LIMIT/OFFSET NULL.
limit = int(limit) if limit else 20
offset = int(offset) if offset else 0
if not search_str and not tag_names and not channel_id and not censored and order in INDEXED_DATE_ORDERS:
# The Videos page with no filters, in date order.
return _search_videos_by_date(order, limit, offset)
params = dict(search_str=search_str, offset=offset)
if channel_id:
wheres.append('v.channel_id = :channel_id')
joins.append('LEFT JOIN channel c ON c.id = v.channel_id')
join_video = True
params['channel_id'] = channel_id
# `deep` searches d_text (captions) as well; the default abc search is much smaller and faster.
# Rank/snippets come from a subquery so they stay valid if a caller adds window functions.
fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \
if search_str else None
if fts_search:
select_columns = f'fg.id, {fts_search.rank_select}'
joins.append(fts_search.join)
params.update(fts_search.params)
else:
select_columns = 'fg.id'
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
if fts_search and headline:
# The title headline is computed in Python by `handle_file_group_search_results`.
headline_selects = ', fts.b_headline, fts.c_headline, fts.d_headline'
else:
headline_selects = ''
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
# whitelist.
order_by = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
null_filter = None
if order:
try:
order_by = VIDEO_ORDERS[order]
except KeyError:
raise
if order in NO_NULL_ORDERS:
null_filter = NO_NULL_ORDERS[order]
wheres.append(null_filter)
if order in JOIN_ORDERS:
join_video = True
effective_order = order or DEFAULT_VIDEO_ORDER
if not fts_search and effective_order in ('rank', '-rank'):
# Rank without a search used to ORDER BY the windowed total (a constant). Equivalent: id.
order_by = 'fg.id ASC' if effective_order == '-rank' else 'fg.id DESC'
if join_video:
joins.insert(0, 'LEFT JOIN video v on v.file_group_id = fg.id')
where = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
join = '\n'.join(joins)
stmt = f'''
SELECT
{select_columns}
{headline_selects}
FROM file_group fg
{join}
{where}
ORDER BY {order_by}
LIMIT {int(limit)} OFFSET :offset
'''.strip()
logger.debug(f'{stmt} {params}')
unfiltered = (not search_str and not tag_names and not channel_id and not censored
and not null_filter)
if unfiltered:
total = cached_search_total(search_filter_cache_key('videos'), _count_videos)
else:
if fts_search:
fts_count = fts.file_group_search(search_str, deep=deep)
count_joins = [j for j in joins if 'file_group_fts' not in j]
if fts_count:
count_joins.append(fts_count.join)
count_wheres = [w for w in wheres]
if fts_count:
count_wheres.append(fts_count.where)
count_join = '\n'.join(count_joins)
count_where = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {count_join} {count_where}'
count_params = dict(params)
if fts_count:
count_params.update(fts_count.params)
else:
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where}'
count_params = params
cache_key = search_filter_cache_key(
'videos', search_str=search_str, channel_id=channel_id, tag_names=tag_names,
censored=censored, deep=deep, null_filter=null_filter,
)
total = cached_search_total(cache_key, lambda: count_file_groups(count_stmt, count_params))
results, total = handle_file_group_search_results(stmt, params, total=total)
return results, total
def download_video_info_json(url: str) -> dict:
"""Download video info JSON, using encrypted cookies for authentication if available."""
ydl_opts = dict(
getcomments=True,
skip_download=True,
extractor_args={'youtube': {'max_comments': ['all', '20', 'all', '10'], 'comment_sort': ['top']}},
cachedir=YTDLP_CACHE_DIR,
**get_yt_dlp_sleep_opts(),
)
ydl_logger = logger.getChild('youtube-dl')
if cookies_unlocked():
logger.info(f'Using encrypted cookies for video info: {url}')
with cookies_for_download() as cookies_path:
ydl_opts['cookiefile'] = str(cookies_path)
http_headers = get_yt_dlp_http_headers()
if http_headers:
ydl_opts['http_headers'] = http_headers
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.params['logger'] = ydl_logger
info = ydl.extract_info(url, download=False)
ydl.sanitize_info(info)
return info
logger.info(f'No cookies available for video info: {url}')
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.params['logger'] = ydl_logger
info = ydl.extract_info(url, download=False)
ydl.sanitize_info(info)
return info
@limit_concurrent(1)
@wrol_mode_check
async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
"""
Fetches Video info json for videos without comments, if comments are found then the info json file is replaced.
Uses encrypted cookies for authentication if available.
"""
one_month_ago = now() - timedelta(days=30)
# Get any videos over a month old that do not have comments.
with get_db_session() as session:
videos = session.query(Video).filter(
# Have yet to get comments.
Video.have_comments != True,
Video.comments_failed != True,
# Needs to be downloadable.
FileGroup.url != None, # noqa
FileGroup.censored != True,
# We want old videos (time for comments to accumulate), or those which we don't know the published date.
or_(
one_month_ago > FileGroup.published_datetime,
FileGroup.published_datetime == None,
),
# Do not download a Videos with a Channel and Channel.download_missing_data is False.
or_(
Channel.download_missing_data == True,
Channel.id == None,
)
).join(FileGroup) \
.outerjoin(Channel) \
.order_by(FileGroup.published_datetime.nullsfirst()) \
.limit(limit)
video_urls = [i.file_group.url for i in videos]
logger.info(f'Found {len(video_urls)} videos missing comments')
def add_video_to_skip_list(video_url: str, censored: bool = False):
"""Do not attempt to get comments of the provided Video again."""
with get_db_session(commit=True) as session_:
for video_, fg in session_.query(Video, FileGroup).join(FileGroup).filter_by(url=video_url).all():
video_.comments_failed = True
fg.censored = censored
download_manager.add_to_skip_list(video_url)
logger.debug(f'add_video_to_skip_list: {video_url=} {censored=}')
for url in video_urls:
# Sleep to catch cancel.
await asyncio.sleep(0)
if download_manager.is_skipped(url):
# This video is skipped, do not try to get comments again.
add_video_to_skip_list(url)
if download_manager.is_disabled or download_manager.is_stopped:
logger.info('Downloads disabled, stop getting comments...')
return
with get_db_session(commit=True) as session:
have_comments = False
for video in session.query(Video).join(FileGroup).filter_by(url=url).all():
if (comments := video.get_comments()) and len(comments) >= 5:
# We already have enough comments. DB may have been wiped, we don't want to fetch comments again.
logger.info(f'Already have comments for Video: {url=}')
have_comments = video.have_comments = True
if have_comments:
download_manager.add_to_skip_list(url)
continue
logger.info(f'Getting comments for: {url}')
try:
# Get info json about the video.
info = download_video_info_json(url)
if not info or not isinstance(info.get('comments'), list):
logger.error(f'Unable to get comments for video: {url=}')
add_video_to_skip_list(url)
continue
except Exception as e:
if 'no longer available' in str(e):
logger.error(f'Giving up on downloading comments because video does not exist: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
elif 'Comments are turned off' in str(e):
logger.error(f'Giving up on downloading comments because comments are disabled: {url=}', exc_info=e)
add_video_to_skip_list(url)
elif 'Private video' in str(e):
logger.error(f'Giving up on downloading comments because video is private: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
elif 'Video unavailable' in str(e):
logger.error(f'Giving up on downloading comments because video is unavailable: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
elif 'video has been removed' in str(e):
logger.error(f'Giving up on downloading comments because video is removed: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
elif ' members' in str(e):
logger.error(f'Giving up on downloading comments because video is members-only: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
elif 'rumble.com' in url and "'bool' object has no attribute 'get'" in str(e):
logger.error(f'Rumble error in yt-dlp', exc_info=e)
add_video_to_skip_list(url, censored=False)
elif 'not a valid URL' in str(e):
logger.error(f'URL is not valid: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
elif ' Sign in ' in str(e):
logger.error(f'Video requires sign-in: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
else:
logger.error(f'Got error when attempting to download video comments: {url=}', exc_info=e)
continue
logger.info(f'Got {len(info["comments"])} comments for Video: {url=}')
with get_db_session(commit=True) as session:
videos = list(session.query(Video).join(FileGroup).filter_by(url=url).all())
for video in videos:
if video.video_path.is_file():
video.replace_info_json(info)
video.have_comments = True
video.comments_failed = False
else:
logger.error(f'Attempting to replace comments for non-existent video! {video}')
if len(videos) == 1:
download_manager.add_to_skip_list(url)
video.comments_failed = True
# Sleep a random amount of time, so we don't spam.
await asyncio.sleep(random.randint(2, 20))
Reference in a new issue View git blame Copy permalink