Creating background process that fetches comments for Videos that do not yet have comments.

Displaying star for Uploader's comments, heart for favorited comments.

Adding wrolpi help files desktop shortcut to rpi image.

Using zim file names as FileGroup.title.

Removing accidental output commit.
This commit is contained in:
Roland 2024-02-27 18:54:14 -07:00
parent df849ffb42
commit 758ff8e810
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
14 changed files with 263 additions and 127 deletions

View file

@ -69,7 +69,7 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
resource_class: small
resource_class: medium
steps:
- checkout
- run: sudo apt-get update
@ -96,7 +96,7 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
resource_class: small
resource_class: medium
steps:
- checkout
- run: sudo apt-get update
@ -123,7 +123,7 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
resource_class: small
resource_class: medium
steps:
- checkout
- run: sudo apt-get update
@ -150,7 +150,7 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
resource_class: small
resource_class: medium
steps:
- checkout
- run: sudo apt-get update
@ -177,7 +177,7 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
resource_class: small
resource_class: medium
steps:
- checkout
- run: sudo apt-get update

View file

@ -0,0 +1,35 @@
"""Video.have_comments
Revision ID: d96a04eb2785
Revises: 365c985893b8
Create Date: 2024-02-27 16:57:29.831210
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = 'd96a04eb2785'
down_revision = '365c985893b8'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video ADD COLUMN have_comments BOOLEAN DEFAULT FALSE')
session.execute('ALTER TABLE video ADD COLUMN comments_failed BOOLEAN DEFAULT FALSE')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS have_comments')
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS comments_failed')

View file

@ -1653,3 +1653,11 @@ export function getParentDirectory(filePath) {
return parentDirectory;
}
export function MultilineText({text, ...props}) {
return <div {...props}>
{text.split('\n').map((line, index, array) =>
index === array.length - 1 ? line : <p>{line}</p>
)}
</div>
}

View file

@ -10,6 +10,7 @@ import {
humanFileSize,
humanNumber,
isoDatetimeToString,
MultilineText,
PageContainer,
PreviewPath,
useTitle
@ -99,20 +100,33 @@ const NoComments = ({video}) => {
const VideoComment = ({comment, children}) => {
const {t} = React.useContext(ThemeContext);
const favoriteComment = <Icon name='heart'/>;
const likesCount = <React.Fragment>
<Icon name='thumbs up'/> {comment['like_count']}
</React.Fragment>;
const {is_favorited, like_count, author, timestamp, text, author_is_uploader} = comment;
// Author comments and favorited comments are important.
let specialIcon = null;
if (author_is_uploader) {
specialIcon = <Icon name='star' color='green'/>;
} else if (is_favorited) {
specialIcon = <Icon name='heart' color='red'/>;
}
const dateElm = <div {...t}>{isoDatetimeToString(timestamp * 1000)}</div>;
const likesElm = like_count && <div {...t}>
<Icon name='thumbs up'/>{comment['like_count']}
</div>;
return <Comment>
<Comment.Content>
<Comment.Author as='a'><span {...t}>{comment['author']}</span></Comment.Author>
<Comment.Author as='a'>
<span {...t}>{specialIcon}{author}</span>
</Comment.Author>
<Comment.Metadata>
<div {...t}>{comment['is_favorited'] && favoriteComment}</div>
<div {...t}>{isoDatetimeToString(comment['timestamp'] * 1000)}</div>
<div {...t}>{comment['like_count'] && likesCount}</div>
{dateElm}
{likesElm}
</Comment.Metadata>
<Comment.Text {...t}>{comment['text']}</Comment.Text>
<Comment.Text {...t}>
<MultilineText text={text} style={{marginLeft: '0.5em'}}/>
</Comment.Text>
</Comment.Content>
{children && !_.isEmpty(children) &&
<CommentGroup>

18
main.py
View file

@ -11,6 +11,7 @@ from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS
from wrolpi import root_api, admin
from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \
wrol_mode_enabled, cancel_refresh_tasks, set_log_level, background_task, cancel_background_tasks
from wrolpi.dates import Seconds
from wrolpi.downloader import download_manager, import_downloads_config
from wrolpi.root_api import api_app
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
@ -201,6 +202,23 @@ async def startup(app: Sanic):
from modules.zim.lib import flag_outdated_zim_files
flag_outdated_zim_files()
from modules.videos.video.lib import get_missing_videos_comments
async def periodic_start_video_missing_comments_download():
async with flags.refresh_complete.wait_for():
# We can't search for Videos missing comments until the refresh has completed.
pass
while True:
# Fetch comments for videos every hour.
if download_manager.disabled.is_set() or download_manager.stopped.is_set():
await asyncio.sleep(10)
else:
await get_missing_videos_comments()
await asyncio.sleep(int(Seconds.hour))
background_task(periodic_start_video_missing_comments_download())
@api_app.after_server_start
async def periodic_downloads(app: Sanic):

View file

@ -2,7 +2,6 @@
import json
import pathlib
import re
import tempfile
import traceback
from abc import ABC
from typing import Tuple, List, Dict
@ -20,7 +19,7 @@ from wrolpi.db import get_db_session
from wrolpi.db import optional_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.lib import glob_shared_stem
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
@ -30,6 +29,7 @@ from .lib import get_downloader_config
from .models import Video, Channel
from .normalize_video_url import normalize_video_url
from .schema import ChannelPostRequest
from .video.lib import download_video_info_json
logger = logger.getChild(__name__)
ydl_logger = logger.getChild('youtube-dl')
@ -469,7 +469,8 @@ class VideoDownloader(Downloader, ABC):
logger.debug(f'Downloading {url} to {out_dir}')
return final_filename, entry
async def download_info_json(self, download: Download, video_path: pathlib.Path) -> DownloadResult:
@staticmethod
async def download_info_json(download: Download, video_path: pathlib.Path) -> DownloadResult:
"""User has requested to refresh Video metadata. Download only metadata files to temporary directory.
Replace existing metadata files only after successful download."""
url = download.url
@ -489,104 +490,25 @@ class VideoDownloader(Downloader, ABC):
if video.channel_id:
location = f'/videos/channel/{video.channel_id}/video/{video.id}'
# Download metadata files into temporary directory so yt-dlp always downloads the latest files.
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_dir = pathlib.Path(tmp_dir)
# Download video info json directly using yt-dlp.
try:
info_json = download_video_info_json(url)
except Exception as e:
logger.error('Failed to download video info json', exc_info=e)
return DownloadResult(success=False, error='Failed to download video info json')
# Download the metadata files, but skip the video file.
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
cmd = (
str(YT_DLP_BIN),
'-f', PREFERRED_VIDEO_FORMAT,
'--skip-download', # SKIP THE VIDEO FILE.
'--match-filter', '!is_live', # Do not attempt to download Live videos.
'--write-info-json',
'--merge-output-format', PREFERRED_VIDEO_EXTENSION,
'--remux-video', PREFERRED_VIDEO_EXTENSION,
'-o', file_name_format,
'--no-cache-dir',
'--compat-options', 'no-live-chat',
# Get top 20 comments, 10 replies per parent.
'--write-comments',
'--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
url,
)
return_code, logs, _ = await self.process_runner(url, cmd, tmp_dir)
with get_db_session(commit=True) as session:
# Find the existing video, replace its info json.
video = Video.get_by_url(url, session)
video.replace_info_json(info_json)
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
if video.get_comments():
video.have_comments = True
if return_code != 0:
error = f'{stdout}\n\n\n{stderr}\n\nvideo info json downloader process exited with {return_code}'
return DownloadResult(
success=False,
error=error,
location=location,
)
logger.debug(f'Downloaded video info json {location=}')
logger.info(f'Successfully downloaded video info json {url} {video}')
# Find the info json file in the temporary directory.
info_json_paths = [i for i in tmp_dir.iterdir() if i.name.endswith('.info.json') and i.stat().st_size]
if len(info_json_paths) == 0:
return DownloadResult(
success=False,
error='No info json file was downloaded.',
location=location,
)
elif len(info_json_paths) > 1:
return DownloadResult(
success=False,
error='More than one info json file was downloaded.',
location=location,
)
info_json_path = info_json_paths[0]
if not info_json_path.stat().st_size:
return DownloadResult(
success=False,
error='Video info json file was empty',
location=location,
)
# Reformat new info json file for human reading.
format_json_file(info_json_path)
with get_db_session(commit=True) as session:
# Find the existing video, replace its info json.
video = Video.get_by_url(url, session)
if video.info_json_path and video.info_json_path.is_file():
# Move old file temporarily. Only delete old data if the swap is successful.
logger.debug(f'Replacing info json file: {video.info_json_path}')
tmp_info_json_path = pathlib.Path(f'{video.info_json_path}.tmp')
if tmp_info_json_path.is_file():
if video.info_json_path.stat().st_size:
tmp_info_json_path.unlink()
else:
raise UnrecoverableDownloadError(
f'Temporary info json file already exists: {tmp_info_json_path}')
# Move existing info json file to a temporary location. Move it back if this fails.
video.info_json_path.rename(tmp_info_json_path)
try:
video.info_json_path.write_text(info_json_path.read_text())
tmp_info_json_path.unlink()
except Exception as e:
logger.error('Failed to swap video info json file', exc_info=e)
tmp_info_json_path.rename(video.info_json_path)
raise
else:
# info json file does not exist, write one next to the video.
video_path = video.video_path
stem, _ = split_path_stem_and_suffix(video_path)
new_info_json_path = pathlib.Path(f'{stem}.info.json')
new_info_json_path.write_text(info_json_path.read_text())
video.file_group.append_files(new_info_json_path)
logger.debug(f'Downloaded video info json {location=}')
logger.info(f'Successfully downloaded video info json {url} {video}')
return DownloadResult(success=True, location=location)
return DownloadResult(success=True, location=location)
channel_downloader = ChannelDownloader()

View file

@ -30,6 +30,8 @@ class Video(ModelHelper, Base):
source_id = Column(String) # The id from yt-dlp
view_count = Column(Integer) # The view count from the ChannelDownloader (or from initial download)
ffprobe_json = deferred(Column(JSON)) # Data that is fetched once from ffprobe (ffmpeg)
have_comments = Column(Boolean, default=False) # see `get_missing_videos_comments`
comments_failed = Column(Boolean, default=False) # see `get_missing_videos_comments`
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
@ -60,8 +62,6 @@ class Video(ModelHelper, Base):
except Exception as e:
logger.error(f'{self} ffprobe_json is invalid', exc_info=e)
info_json = self.get_info_json() or dict()
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
d['video'] = dict(
caption=self.file_group.d_text,
@ -70,7 +70,7 @@ class Video(ModelHelper, Base):
channel_id=self.channel_id,
codec_names=codec_names,
codec_types=codec_types,
comments=info_json.get('comments'),
comments=self.get_comments(),
description=self.file_group.c_text or self.get_video_description(),
id=self.id,
info_json_file=self.info_json_file,
@ -102,7 +102,7 @@ class Video(ModelHelper, Base):
logger.warning(f'{self} cannot be added to skip list because it does not have a URL')
def get_info_json(self) -> Optional[Dict]:
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
"""If this Video has an info_json file, return its contents. Otherwise, return None."""
info_json_path = self.info_json_path
if not info_json_path:
return
@ -121,12 +121,8 @@ class Video(ModelHelper, Base):
"""
Get the Video description from the file system.
"""
# First try to get description from info_json file.
info_json = self.get_info_json()
if info_json:
description = info_json.get('description')
if description:
return description
if (info_json := self.get_info_json()) and (description := info_json.get('description')):
return description
def get_surrounding_videos(self):
"""
@ -231,6 +227,9 @@ class Video(ModelHelper, Base):
self.file_group.c_text = self.get_video_description()
# self.file_group.d_text is handled in `validate_video`.
if (comments := self.get_comments()) and len(comments) >= 5:
self.have_comments = True
@staticmethod
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
file_group = FileGroup.from_paths(session, *paths)
@ -477,6 +476,17 @@ class Video(ModelHelper, Base):
return f'/videos/channel/{self.channel_id}/video/{self.id}'
return f'/videos/video/{self.id}'
def get_comments(self):
return (self.get_info_json() or dict()).get('comments')
def replace_info_json(self, info_json: dict):
"""Replace the info json file with the new json dict. Handles adding new info_json file, if necessary."""
info_json_path = self.info_json_path or self.video_path.with_suffix('.info.json')
info_json_path.write_text(json.dumps(info_json, indent=2))
if not self.info_json_path:
self.file_group.append_files(info_json_path)
class Channel(ModelHelper, Base):
__tablename__ = 'channel'

View file

@ -1,12 +1,22 @@
import asyncio
import json
import random
from datetime import timedelta
from typing import Tuple, Optional, List, Dict
import yt_dlp
from sqlalchemy import or_
from sqlalchemy.orm import Session
from modules.videos.models import Video
from wrolpi.common import run_after, logger
from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check
from wrolpi.dates import now
from wrolpi.db import get_db_session, optional_session
from wrolpi.downloader import download_manager
from wrolpi.files.lib import handle_file_group_search_results
from wrolpi.files.models import FileGroup
from wrolpi.tags import tag_append_sub_select_where
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT
from ..errors import UnknownVideo
from ..lib import save_channels_config
@ -157,3 +167,100 @@ def delete_videos(*video_ids: int, session: Session = None):
for video in videos:
video.delete()
session.commit()
def download_video_info_json(url: str) -> dict:
ydl_opts = dict(
getcomments=True,
skip_download=True,
extractor_args={'youtube': {'max_comments': ['all', '20', 'all', '10'], 'comment_sort': ['top']}},
)
ydl_logger = logger.getChild('youtube-dl')
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.params['logger'] = ydl_logger
info = ydl.extract_info(url, download=False)
ydl.sanitize_info(info)
return info
@limit_concurrent(1)
@wrol_mode_check
async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
"""
Fetches Video info json for videos without comments, if comments are found then the info json file is replaced.
"""
one_month_ago = now() - timedelta(days=30)
# Get any videos over a month old that do not have comments.
with get_db_session() as session:
videos = session.query(Video).filter(
# Have yet to get comments.
Video.have_comments == False,
Video.comments_failed == False,
# Need a URL to try to get comments.
FileGroup.url != None, # noqa
# We want old videos (time for comments to accumulate), or those which we don't know the published date.
or_(
one_month_ago > FileGroup.published_datetime,
FileGroup.published_datetime == None,
),
).join(FileGroup) \
.order_by(FileGroup.published_datetime.nullsfirst()) \
.limit(limit)
video_urls = [i.file_group.url for i in videos]
logger.info(f'Found {len(video_urls)} videos missing comments')
def add_video_to_skip_list(video_url: str):
"""Do not attempt to get comments of the provided Video again."""
with get_db_session(commit=True) as session_:
for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all():
video_.comments_failed = True
for url in video_urls:
# Sleep to catch cancel.
await asyncio.sleep(0)
if download_manager.is_skipped(url):
# This video is skipped, do not try to get comments again.
add_video_to_skip_list(url)
with get_db_session(commit=True) as session:
have_comments = False
for video in session.query(Video).join(FileGroup).filter_by(url=url).all():
if (comments := video.get_comments()) and len(comments) >= 5:
# We already have enough comments. DB may have been wiped, we don't want to fetch comments again.
logger.info(f'Already have comments for Video: {url=}')
have_comments = video.have_comments = True
if have_comments:
download_manager.add_to_skip_list(url)
continue
tries = 5
while tries > 0:
try:
tries -= 1
# Get info json about the video. Retry until we have a list of comments.
info = download_video_info_json(url)
if info and isinstance(info.get('comments'), list):
break
except Exception as e:
logger.error(f'Got error when attempting to download comments: {url=}', exc_info=e)
else:
logger.error(f'Never got comments for video: {url=}')
download_manager.add_to_skip_list(url)
add_video_to_skip_list(url)
continue
logger.debug(f'Got {len(info["comments"])} comments for Video: {url=}')
with get_db_session(commit=True) as session:
for video in session.query(Video).join(FileGroup).filter_by(url=url).all():
video.replace_info_json(info)
video.have_comments = True
# Sleep a random amount of time, so we don't spam.
await asyncio.sleep(random.randint(2, 20))

View file

@ -85,7 +85,7 @@ def zim_path_factory(test_zim_bytes, test_directory):
@pytest.fixture
def test_zim(test_session, zim_path_factory, test_directory) -> Zim:
def test_zim(test_session, zim_path_factory) -> Zim:
zim = Zim.from_paths(test_session, zim_path_factory())
return zim

View file

@ -19,7 +19,7 @@ from wrolpi.common import register_modeler, logger, extract_html_text, extract_h
register_refresh_cleanup, background_task
from wrolpi.db import get_db_session, optional_session, get_db_curs
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import refresh_files
from wrolpi.files.lib import refresh_files, split_file_name_words
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, DOCKERIZED
@ -62,6 +62,8 @@ async def zim_modeler():
session.add(zim)
session.flush([zim])
zim_id = zim.id
file_group.title = file_group.primary_path.name
file_group.a_text = split_file_name_words(file_group.primary_path.name)
file_group.indexed = True
except Exception as e:
if PYTEST:

View file

@ -328,3 +328,16 @@ async def test_zim_subscription_download_import(test_session, test_downloader_co
assert subscription.download_id == recurring.id
assert subscription.name == 'Wikisource'
assert subscription.language == 'en'
@pytest.mark.asyncio
async def test_zim_modeler(test_session, zim_factory):
"""Zim modeler sets FileGroup title and a_text."""
zim_factory('the_zim_title')
test_session.commit()
await files_lib.refresh_files()
fg: FileGroup = test_session.query(FileGroup).one()
assert fg.title == 'the_zim_title'
assert fg.a_text == 'the zim title'

View file

@ -32,6 +32,14 @@ Type=Link
URL=http://127.0.0.1:8086
Icon=/opt/wrolpi-help/venv/lib/python3.11/site-packages/mkdocs/themes/mkdocs/img/favicon.ico
EOF
cat >/home/pi/Desktop/wrolpi-help-files.desktop <<'EOF'
[Desktop Entry]
Encoding=UTF-8
Name=WROLPi Help Files
Type=Link
URL=/opt/wrolpi-help/docs/
Icon=/usr/share/icons/gnome/48x48/places/folder.png
EOF
chown -R pi:pi /home/pi
chmod 0600 /home/pi/.pgpass
usermod -aG wrolpi pi

View file

@ -80,9 +80,6 @@ def get_db_context() -> Tuple[sqlalchemy.create_engine, Session]:
return local_engine, session
# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
@contextmanager
def get_db_session(commit: bool = False) -> ContextManager[Session]:
"""

View file

@ -75,6 +75,8 @@ FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100))
FILE_MAX_PDF_SIZE = int(os.environ.get('FILE_MAX_PDF_SIZE', 40_000_000))
FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000))
VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 50))
DEFAULT_HTTP_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36',
}