mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
3967a49989
Cold archive/video browsing took 18-90 seconds on a Pi: file_group rows carry their indexed texts inline (SQLite has no out-of-line TOAST like Postgres), so any plan that reads file_group rows to filter or sort drags gigabytes of text pages through USB. - Browse archives with file_group as the outer table (CROSS JOIN) so SQLite walks the date index and probes archive's covering index; driving from archive probed every file_group row for the sort key. - Tie-break orders with fg.id instead of LOWER(fg.primary_path); the rowid is present in every index, the path expression forced full-row reads (and a full scan on the videos browse). - Add a (mimetype, effective_datetime) covering index so the videos mimetype filter + date order resolve index-only. - PRAGMA optimize after a global refresh; the planner had no sqlite_stat1 at all after the rebuild. Warm timings on 10.0.0.9 (124k archives / 24k videos): archive browse 0.4s -> 0.22s index-only, videos browse 0.38s -> 0.07s; cold I/O drops from ~124k random fat-page probes to a few MB of contiguous index. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
336 lines
14 KiB
Python
336 lines
14 KiB
Python
import asyncio
|
|
import random
|
|
from datetime import timedelta
|
|
from typing import Tuple, Optional, List
|
|
|
|
import yt_dlp
|
|
from sqlalchemy import or_
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modules.videos.models import Video, Channel
|
|
from wrolpi import fts
|
|
from wrolpi.common import logger, limit_concurrent, wrol_mode_check
|
|
from wrolpi.dates import now
|
|
from wrolpi.db import get_db_session
|
|
from wrolpi.downloader import download_manager
|
|
from wrolpi.files.lib import handle_file_group_search_results
|
|
from wrolpi.files.models import FileGroup
|
|
from wrolpi.tags import tag_append_sub_select_where
|
|
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR
|
|
from ..cookies import cookies_unlocked, cookies_for_download
|
|
from ..lib import get_yt_dlp_http_headers, get_yt_dlp_sleep_opts
|
|
|
|
logger.getChild(__name__)
|
|
|
|
|
|
def get_video_for_app(file_group_id: int, skip_viewed: bool = False) -> Tuple[dict, Optional[dict], Optional[dict]]:
|
|
"""
|
|
Get a Video by its FileGroup ID, with its prev/next videos. Mark the Video as viewed.
|
|
"""
|
|
with get_db_session(commit=True) as session:
|
|
video = Video.find_by_file_group_id(session, file_group_id)
|
|
if not skip_viewed:
|
|
video.file_group.set_viewed()
|
|
previous_video, next_video = video.get_surrounding_videos()
|
|
|
|
video = video.__json__()
|
|
previous_video = previous_video.__json__() if previous_video and previous_video.file_group else None
|
|
next_video = next_video.__json__() if next_video and next_video.file_group else None
|
|
|
|
return video, previous_video, next_video
|
|
|
|
|
|
def get_video(file_group_id: int) -> Video:
|
|
"""
|
|
Get a Video by its FileGroup ID.
|
|
"""
|
|
with get_db_session() as session:
|
|
video = Video.find_by_file_group_id(session, file_group_id)
|
|
return video
|
|
|
|
|
|
VIDEO_ORDERS = {
|
|
# fg.id (the rowid) is the pagination tiebreaker: it is present in every index, so these
|
|
# orders stay index-only. LOWER(fg.primary_path) forced SQLite to read every (large) row.
|
|
# Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
|
|
'published_datetime': 'fg.effective_datetime ASC, fg.id ASC',
|
|
'-published_datetime': 'fg.effective_datetime DESC NULLS LAST, fg.id ASC',
|
|
'rank': '2 DESC, fg.id DESC',
|
|
'-rank': '2 ASC, fg.id ASC',
|
|
'size': 'fg.size ASC, fg.id ASC',
|
|
'-size': 'fg.size DESC, fg.id DESC',
|
|
'length': 'fg.length ASC, fg.id ASC',
|
|
'-length': 'fg.length DESC, fg.id DESC',
|
|
'size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END ASC, fg.id ASC',
|
|
'-size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END DESC, fg.id DESC',
|
|
'viewed': 'fg.viewed ASC',
|
|
'-viewed': 'fg.viewed DESC',
|
|
'view_count': 'v.view_count ASC',
|
|
'-view_count': 'v.view_count DESC',
|
|
'download_datetime': 'fg.download_datetime ASC, fg.id ASC',
|
|
'-download_datetime': 'fg.download_datetime DESC NULLS LAST, fg.id ASC',
|
|
}
|
|
NO_NULL_ORDERS = {
|
|
'viewed': 'fg.viewed IS NOT NULL',
|
|
'-viewed': 'fg.viewed IS NOT NULL',
|
|
'length': 'fg.length IS NOT NULL',
|
|
'-length': 'fg.length IS NOT NULL',
|
|
'size': 'fg.size IS NOT NULL',
|
|
'-size': 'fg.size IS NOT NULL',
|
|
'size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
|
|
'-size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
|
|
'view_count': 'v.view_count IS NOT NULL',
|
|
'-view_count': 'v.view_count IS NOT NULL',
|
|
}
|
|
JOIN_ORDERS = ('view_count', '-view_count')
|
|
DEFAULT_VIDEO_ORDER = 'rank'
|
|
VIDEO_QUERY_LIMIT = 24
|
|
|
|
|
|
def search_videos(
|
|
search_str: str = None,
|
|
offset: int = None,
|
|
limit: int = VIDEO_QUERY_LIMIT,
|
|
channel_id: int = None,
|
|
order: str = None,
|
|
tag_names: List[str] = None,
|
|
headline: bool = False,
|
|
censored: bool = False,
|
|
deep: bool = False,
|
|
) -> Tuple[List[dict], int]:
|
|
tag_names = tag_names or []
|
|
# Search videos and audio-only files.
|
|
wheres = ["(fg.mimetype LIKE 'video/%' OR fg.mimetype LIKE 'audio/%')"]
|
|
if censored:
|
|
# Only return videos which are no longer available for download.
|
|
wheres.append('fg.censored = TRUE')
|
|
joins = list()
|
|
join_video = False
|
|
|
|
# Explicit JSON nulls bypass the schema defaults; SQLite rejects LIMIT/OFFSET NULL.
|
|
limit = int(limit) if limit else 20
|
|
params = dict(search_str=search_str, offset=int(offset) if offset else 0)
|
|
if channel_id:
|
|
wheres.append('v.channel_id = :channel_id')
|
|
joins.append('LEFT JOIN channel c ON c.id = v.channel_id')
|
|
join_video = True
|
|
params['channel_id'] = channel_id
|
|
|
|
# `deep` searches d_text (captions) as well; the default abc search is much smaller and faster.
|
|
# The subquery join is required: snippet() cannot coexist with COUNT(*) OVER() in one SELECT.
|
|
fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \
|
|
if search_str else None
|
|
if fts_search:
|
|
# A search_str was provided by the user, modify the query to filter by it.
|
|
select_columns = f'fg.id, {fts_search.rank_select}, COUNT(*) OVER() AS total'
|
|
joins.append(fts_search.join)
|
|
params.update(fts_search.params)
|
|
else:
|
|
# No (usable) search_str provided. Get id and total only.
|
|
select_columns = 'fg.id, COUNT(*) OVER() AS total'
|
|
|
|
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
|
|
|
|
if fts_search and headline:
|
|
# The title headline is computed in Python by `handle_file_group_search_results`.
|
|
headline_selects = ', fts.b_headline, fts.c_headline, fts.d_headline'
|
|
else:
|
|
headline_selects = ''
|
|
|
|
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
|
|
# whitelist.
|
|
order_by = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
|
|
if order:
|
|
try:
|
|
order_by = VIDEO_ORDERS[order]
|
|
except KeyError:
|
|
raise
|
|
if order in NO_NULL_ORDERS:
|
|
wheres.append(NO_NULL_ORDERS[order])
|
|
if order in JOIN_ORDERS:
|
|
join_video = True
|
|
|
|
if join_video:
|
|
joins.insert(0, 'LEFT JOIN video v on v.file_group_id = fg.id')
|
|
|
|
wheres = '\n AND '.join(wheres)
|
|
where = f'WHERE\n{wheres}' if wheres else ''
|
|
join = '\n'.join(joins)
|
|
stmt = f'''
|
|
SELECT
|
|
{select_columns}
|
|
{headline_selects}
|
|
FROM file_group fg
|
|
{join}
|
|
{where}
|
|
ORDER BY {order_by}
|
|
LIMIT {int(limit)} OFFSET :offset
|
|
'''.strip()
|
|
logger.debug(f'{stmt} {params}')
|
|
|
|
results, total = handle_file_group_search_results(stmt, params)
|
|
return results, total
|
|
|
|
|
|
def download_video_info_json(url: str) -> dict:
|
|
"""Download video info JSON, using encrypted cookies for authentication if available."""
|
|
ydl_opts = dict(
|
|
getcomments=True,
|
|
skip_download=True,
|
|
extractor_args={'youtube': {'max_comments': ['all', '20', 'all', '10'], 'comment_sort': ['top']}},
|
|
cachedir=YTDLP_CACHE_DIR,
|
|
**get_yt_dlp_sleep_opts(),
|
|
)
|
|
|
|
ydl_logger = logger.getChild('youtube-dl')
|
|
|
|
if cookies_unlocked():
|
|
logger.info(f'Using encrypted cookies for video info: {url}')
|
|
with cookies_for_download() as cookies_path:
|
|
ydl_opts['cookiefile'] = str(cookies_path)
|
|
http_headers = get_yt_dlp_http_headers()
|
|
if http_headers:
|
|
ydl_opts['http_headers'] = http_headers
|
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
|
ydl.params['logger'] = ydl_logger
|
|
info = ydl.extract_info(url, download=False)
|
|
ydl.sanitize_info(info)
|
|
return info
|
|
|
|
logger.info(f'No cookies available for video info: {url}')
|
|
|
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
|
ydl.params['logger'] = ydl_logger
|
|
info = ydl.extract_info(url, download=False)
|
|
ydl.sanitize_info(info)
|
|
return info
|
|
|
|
|
|
@limit_concurrent(1)
|
|
@wrol_mode_check
|
|
async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
|
|
"""
|
|
Fetches Video info json for videos without comments, if comments are found then the info json file is replaced.
|
|
Uses encrypted cookies for authentication if available.
|
|
"""
|
|
one_month_ago = now() - timedelta(days=30)
|
|
|
|
# Get any videos over a month old that do not have comments.
|
|
with get_db_session() as session:
|
|
videos = session.query(Video).filter(
|
|
# Have yet to get comments.
|
|
Video.have_comments != True,
|
|
Video.comments_failed != True,
|
|
# Needs to be downloadable.
|
|
FileGroup.url != None, # noqa
|
|
FileGroup.censored != True,
|
|
# We want old videos (time for comments to accumulate), or those which we don't know the published date.
|
|
or_(
|
|
one_month_ago > FileGroup.published_datetime,
|
|
FileGroup.published_datetime == None,
|
|
),
|
|
# Do not download a Videos with a Channel and Channel.download_missing_data is False.
|
|
or_(
|
|
Channel.download_missing_data == True,
|
|
Channel.id == None,
|
|
)
|
|
).join(FileGroup) \
|
|
.outerjoin(Channel) \
|
|
.order_by(FileGroup.published_datetime.nullsfirst()) \
|
|
.limit(limit)
|
|
video_urls = [i.file_group.url for i in videos]
|
|
|
|
logger.info(f'Found {len(video_urls)} videos missing comments')
|
|
|
|
def add_video_to_skip_list(video_url: str, censored: bool = False):
|
|
"""Do not attempt to get comments of the provided Video again."""
|
|
with get_db_session(commit=True) as session_:
|
|
for video_, fg in session_.query(Video, FileGroup).join(FileGroup).filter_by(url=video_url).all():
|
|
video_.comments_failed = True
|
|
fg.censored = censored
|
|
download_manager.add_to_skip_list(video_url)
|
|
logger.debug(f'add_video_to_skip_list: {video_url=} {censored=}')
|
|
|
|
for url in video_urls:
|
|
# Sleep to catch cancel.
|
|
await asyncio.sleep(0)
|
|
|
|
if download_manager.is_skipped(url):
|
|
# This video is skipped, do not try to get comments again.
|
|
add_video_to_skip_list(url)
|
|
|
|
if download_manager.is_disabled or download_manager.is_stopped:
|
|
logger.info('Downloads disabled, stop getting comments...')
|
|
return
|
|
|
|
with get_db_session(commit=True) as session:
|
|
have_comments = False
|
|
for video in session.query(Video).join(FileGroup).filter_by(url=url).all():
|
|
if (comments := video.get_comments()) and len(comments) >= 5:
|
|
# We already have enough comments. DB may have been wiped, we don't want to fetch comments again.
|
|
logger.info(f'Already have comments for Video: {url=}')
|
|
have_comments = video.have_comments = True
|
|
|
|
if have_comments:
|
|
download_manager.add_to_skip_list(url)
|
|
continue
|
|
|
|
logger.info(f'Getting comments for: {url}')
|
|
|
|
try:
|
|
# Get info json about the video.
|
|
info = download_video_info_json(url)
|
|
|
|
if not info or not isinstance(info.get('comments'), list):
|
|
logger.error(f'Unable to get comments for video: {url=}')
|
|
add_video_to_skip_list(url)
|
|
continue
|
|
except Exception as e:
|
|
if 'no longer available' in str(e):
|
|
logger.error(f'Giving up on downloading comments because video does not exist: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url, censored=True)
|
|
elif 'Comments are turned off' in str(e):
|
|
logger.error(f'Giving up on downloading comments because comments are disabled: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url)
|
|
elif 'Private video' in str(e):
|
|
logger.error(f'Giving up on downloading comments because video is private: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url, censored=True)
|
|
elif 'Video unavailable' in str(e):
|
|
logger.error(f'Giving up on downloading comments because video is unavailable: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url, censored=True)
|
|
elif 'video has been removed' in str(e):
|
|
logger.error(f'Giving up on downloading comments because video is removed: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url, censored=True)
|
|
elif ' members' in str(e):
|
|
logger.error(f'Giving up on downloading comments because video is members-only: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url, censored=True)
|
|
elif 'rumble.com' in url and "'bool' object has no attribute 'get'" in str(e):
|
|
logger.error(f'Rumble error in yt-dlp', exc_info=e)
|
|
add_video_to_skip_list(url, censored=False)
|
|
elif 'not a valid URL' in str(e):
|
|
logger.error(f'URL is not valid: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url, censored=True)
|
|
elif ' Sign in ' in str(e):
|
|
logger.error(f'Video requires sign-in: {url=}', exc_info=e)
|
|
add_video_to_skip_list(url, censored=True)
|
|
else:
|
|
logger.error(f'Got error when attempting to download video comments: {url=}', exc_info=e)
|
|
continue
|
|
|
|
logger.info(f'Got {len(info["comments"])} comments for Video: {url=}')
|
|
|
|
with get_db_session(commit=True) as session:
|
|
videos = list(session.query(Video).join(FileGroup).filter_by(url=url).all())
|
|
for video in videos:
|
|
if video.video_path.is_file():
|
|
video.replace_info_json(info)
|
|
video.have_comments = True
|
|
video.comments_failed = False
|
|
else:
|
|
logger.error(f'Attempting to replace comments for non-existent video! {video}')
|
|
if len(videos) == 1:
|
|
download_manager.add_to_skip_list(url)
|
|
video.comments_failed = True
|
|
|
|
# Sleep a random amount of time, so we don't spam.
|
|
await asyncio.sleep(random.randint(2, 20))
|