diff --git a/.circleci/config.yml b/.circleci/config.yml index e7946204..acb7e620 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -96,7 +96,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -123,7 +123,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -150,7 +150,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -177,7 +177,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update diff --git a/alembic/versions/d96a04eb2785_video_have_comments.py b/alembic/versions/d96a04eb2785_video_have_comments.py new file mode 100644 index 00000000..f29aa1af --- /dev/null +++ b/alembic/versions/d96a04eb2785_video_have_comments.py @@ -0,0 +1,35 @@ +"""Video.have_comments + +Revision ID: d96a04eb2785 +Revises: 365c985893b8 +Create Date: 2024-02-27 16:57:29.831210 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'd96a04eb2785' +down_revision = '365c985893b8' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video ADD COLUMN have_comments BOOLEAN DEFAULT FALSE') + session.execute('ALTER TABLE video ADD COLUMN comments_failed BOOLEAN DEFAULT FALSE') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS have_comments') + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS comments_failed') diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 0bea8ef8..f9ac7288 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -1653,3 +1653,11 @@ export function getParentDirectory(filePath) { return parentDirectory; } + +export function MultilineText({text, ...props}) { + return
+ {text.split('\n').map((line, index, array) => + index === array.length - 1 ? line :

{line}

+ )} +
+} diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 555fbcb4..ecb6473f 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -10,6 +10,7 @@ import { humanFileSize, humanNumber, isoDatetimeToString, + MultilineText, PageContainer, PreviewPath, useTitle @@ -99,20 +100,33 @@ const NoComments = ({video}) => { const VideoComment = ({comment, children}) => { const {t} = React.useContext(ThemeContext); - const favoriteComment = ; - const likesCount = - {comment['like_count']} - ; + const {is_favorited, like_count, author, timestamp, text, author_is_uploader} = comment; + + // Author comments and favorited comments are important. + let specialIcon = null; + if (author_is_uploader) { + specialIcon = ; + } else if (is_favorited) { + specialIcon = ; + } + + const dateElm =
{isoDatetimeToString(timestamp * 1000)}
; + const likesElm = like_count &&
+ {comment['like_count']} +
; return - {comment['author']} + + {specialIcon}{author} + -
{comment['is_favorited'] && favoriteComment}
-
{isoDatetimeToString(comment['timestamp'] * 1000)}
-
{comment['like_count'] && likesCount}
+ {dateElm} + {likesElm}
- {comment['text']} + + +
{children && !_.isEmpty(children) && diff --git a/main.py b/main.py index 9bd19c3d..3f3d2df6 100755 --- a/main.py +++ b/main.py @@ -11,6 +11,7 @@ from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS from wrolpi import root_api, admin from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \ wrol_mode_enabled, cancel_refresh_tasks, set_log_level, background_task, cancel_background_tasks +from wrolpi.dates import Seconds from wrolpi.downloader import download_manager, import_downloads_config from wrolpi.root_api import api_app from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST @@ -201,6 +202,23 @@ async def startup(app: Sanic): from modules.zim.lib import flag_outdated_zim_files flag_outdated_zim_files() + from modules.videos.video.lib import get_missing_videos_comments + + async def periodic_start_video_missing_comments_download(): + async with flags.refresh_complete.wait_for(): + # We can't search for Videos missing comments until the refresh has completed. + pass + + while True: + # Fetch comments for videos every hour. + if download_manager.disabled.is_set() or download_manager.stopped.is_set(): + await asyncio.sleep(10) + else: + await get_missing_videos_comments() + await asyncio.sleep(int(Seconds.hour)) + + background_task(periodic_start_video_missing_comments_download()) + @api_app.after_server_start async def periodic_downloads(app: Sanic): diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 2785fb0d..6286e050 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -2,7 +2,6 @@ import json import pathlib import re -import tempfile import traceback from abc import ABC from typing import Tuple, List, Dict @@ -20,7 +19,7 @@ from wrolpi.db import get_db_session from wrolpi.db import optional_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError -from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix +from wrolpi.files.lib import glob_shared_stem from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel @@ -30,6 +29,7 @@ from .lib import get_downloader_config from .models import Video, Channel from .normalize_video_url import normalize_video_url from .schema import ChannelPostRequest +from .video.lib import download_video_info_json logger = logger.getChild(__name__) ydl_logger = logger.getChild('youtube-dl') @@ -469,7 +469,8 @@ class VideoDownloader(Downloader, ABC): logger.debug(f'Downloading {url} to {out_dir}') return final_filename, entry - async def download_info_json(self, download: Download, video_path: pathlib.Path) -> DownloadResult: + @staticmethod + async def download_info_json(download: Download, video_path: pathlib.Path) -> DownloadResult: """User has requested to refresh Video metadata. Download only metadata files to temporary directory. Replace existing metadata files only after successful download.""" url = download.url @@ -489,104 +490,25 @@ class VideoDownloader(Downloader, ABC): if video.channel_id: location = f'/videos/channel/{video.channel_id}/video/{video.id}' - # Download metadata files into temporary directory so yt-dlp always downloads the latest files. - with tempfile.TemporaryDirectory() as tmp_dir: - tmp_dir = pathlib.Path(tmp_dir) + # Download video info json directly using yt-dlp. + try: + info_json = download_video_info_json(url) + except Exception as e: + logger.error('Failed to download video info json', exc_info=e) + return DownloadResult(success=False, error='Failed to download video info json') - # Download the metadata files, but skip the video file. - file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' - cmd = ( - str(YT_DLP_BIN), - '-f', PREFERRED_VIDEO_FORMAT, - '--skip-download', # SKIP THE VIDEO FILE. - '--match-filter', '!is_live', # Do not attempt to download Live videos. - '--write-info-json', - '--merge-output-format', PREFERRED_VIDEO_EXTENSION, - '--remux-video', PREFERRED_VIDEO_EXTENSION, - '-o', file_name_format, - '--no-cache-dir', - '--compat-options', 'no-live-chat', - # Get top 20 comments, 10 replies per parent. - '--write-comments', - '--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top', - url, - ) - return_code, logs, _ = await self.process_runner(url, cmd, tmp_dir) + with get_db_session(commit=True) as session: + # Find the existing video, replace its info json. + video = Video.get_by_url(url, session) + video.replace_info_json(info_json) - stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr'] - stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout'] + if video.get_comments(): + video.have_comments = True - if return_code != 0: - error = f'{stdout}\n\n\n{stderr}\n\nvideo info json downloader process exited with {return_code}' - return DownloadResult( - success=False, - error=error, - location=location, - ) + logger.debug(f'Downloaded video info json {location=}') + logger.info(f'Successfully downloaded video info json {url} {video}') - # Find the info json file in the temporary directory. - info_json_paths = [i for i in tmp_dir.iterdir() if i.name.endswith('.info.json') and i.stat().st_size] - if len(info_json_paths) == 0: - return DownloadResult( - success=False, - error='No info json file was downloaded.', - location=location, - ) - elif len(info_json_paths) > 1: - return DownloadResult( - success=False, - error='More than one info json file was downloaded.', - location=location, - ) - - info_json_path = info_json_paths[0] - - if not info_json_path.stat().st_size: - return DownloadResult( - success=False, - error='Video info json file was empty', - location=location, - ) - - # Reformat new info json file for human reading. - format_json_file(info_json_path) - - with get_db_session(commit=True) as session: - # Find the existing video, replace its info json. - video = Video.get_by_url(url, session) - - if video.info_json_path and video.info_json_path.is_file(): - # Move old file temporarily. Only delete old data if the swap is successful. - logger.debug(f'Replacing info json file: {video.info_json_path}') - tmp_info_json_path = pathlib.Path(f'{video.info_json_path}.tmp') - if tmp_info_json_path.is_file(): - if video.info_json_path.stat().st_size: - tmp_info_json_path.unlink() - else: - raise UnrecoverableDownloadError( - f'Temporary info json file already exists: {tmp_info_json_path}') - - # Move existing info json file to a temporary location. Move it back if this fails. - video.info_json_path.rename(tmp_info_json_path) - try: - video.info_json_path.write_text(info_json_path.read_text()) - tmp_info_json_path.unlink() - except Exception as e: - logger.error('Failed to swap video info json file', exc_info=e) - tmp_info_json_path.rename(video.info_json_path) - raise - else: - # info json file does not exist, write one next to the video. - video_path = video.video_path - stem, _ = split_path_stem_and_suffix(video_path) - new_info_json_path = pathlib.Path(f'{stem}.info.json') - new_info_json_path.write_text(info_json_path.read_text()) - video.file_group.append_files(new_info_json_path) - - logger.debug(f'Downloaded video info json {location=}') - logger.info(f'Successfully downloaded video info json {url} {video}') - - return DownloadResult(success=True, location=location) + return DownloadResult(success=True, location=location) channel_downloader = ChannelDownloader() diff --git a/modules/videos/models.py b/modules/videos/models.py index dc38f219..f27f81ca 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -30,6 +30,8 @@ class Video(ModelHelper, Base): source_id = Column(String) # The id from yt-dlp view_count = Column(Integer) # The view count from the ChannelDownloader (or from initial download) ffprobe_json = deferred(Column(JSON)) # Data that is fetched once from ffprobe (ffmpeg) + have_comments = Column(Boolean, default=False) # see `get_missing_videos_comments` + comments_failed = Column(Boolean, default=False) # see `get_missing_videos_comments` channel_id = Column(Integer, ForeignKey('channel.id')) channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos') @@ -60,8 +62,6 @@ class Video(ModelHelper, Base): except Exception as e: logger.error(f'{self} ffprobe_json is invalid', exc_info=e) - info_json = self.get_info_json() or dict() - # Put live data in "video" instead of "data" to avoid confusion on the frontend. d['video'] = dict( caption=self.file_group.d_text, @@ -70,7 +70,7 @@ class Video(ModelHelper, Base): channel_id=self.channel_id, codec_names=codec_names, codec_types=codec_types, - comments=info_json.get('comments'), + comments=self.get_comments(), description=self.file_group.c_text or self.get_video_description(), id=self.id, info_json_file=self.info_json_file, @@ -102,7 +102,7 @@ class Video(ModelHelper, Base): logger.warning(f'{self} cannot be added to skip list because it does not have a URL') def get_info_json(self) -> Optional[Dict]: - """If this Video has an info_json file, return it's contents. Otherwise, return None.""" + """If this Video has an info_json file, return its contents. Otherwise, return None.""" info_json_path = self.info_json_path if not info_json_path: return @@ -121,12 +121,8 @@ class Video(ModelHelper, Base): """ Get the Video description from the file system. """ - # First try to get description from info_json file. - info_json = self.get_info_json() - if info_json: - description = info_json.get('description') - if description: - return description + if (info_json := self.get_info_json()) and (description := info_json.get('description')): + return description def get_surrounding_videos(self): """ @@ -231,6 +227,9 @@ class Video(ModelHelper, Base): self.file_group.c_text = self.get_video_description() # self.file_group.d_text is handled in `validate_video`. + if (comments := self.get_comments()) and len(comments) >= 5: + self.have_comments = True + @staticmethod def from_paths(session: Session, *paths: pathlib.Path) -> 'Video': file_group = FileGroup.from_paths(session, *paths) @@ -477,6 +476,17 @@ class Video(ModelHelper, Base): return f'/videos/channel/{self.channel_id}/video/{self.id}' return f'/videos/video/{self.id}' + def get_comments(self): + return (self.get_info_json() or dict()).get('comments') + + def replace_info_json(self, info_json: dict): + """Replace the info json file with the new json dict. Handles adding new info_json file, if necessary.""" + info_json_path = self.info_json_path or self.video_path.with_suffix('.info.json') + info_json_path.write_text(json.dumps(info_json, indent=2)) + + if not self.info_json_path: + self.file_group.append_files(info_json_path) + class Channel(ModelHelper, Base): __tablename__ = 'channel' diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index bef46ed5..63be8cbb 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -1,12 +1,22 @@ +import asyncio +import json +import random +from datetime import timedelta from typing import Tuple, Optional, List, Dict +import yt_dlp +from sqlalchemy import or_ from sqlalchemy.orm import Session from modules.videos.models import Video -from wrolpi.common import run_after, logger +from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check +from wrolpi.dates import now from wrolpi.db import get_db_session, optional_session +from wrolpi.downloader import download_manager from wrolpi.files.lib import handle_file_group_search_results +from wrolpi.files.models import FileGroup from wrolpi.tags import tag_append_sub_select_where +from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT from ..errors import UnknownVideo from ..lib import save_channels_config @@ -157,3 +167,100 @@ def delete_videos(*video_ids: int, session: Session = None): for video in videos: video.delete() session.commit() + + +def download_video_info_json(url: str) -> dict: + ydl_opts = dict( + getcomments=True, + skip_download=True, + extractor_args={'youtube': {'max_comments': ['all', '20', 'all', '10'], 'comment_sort': ['top']}}, + ) + + ydl_logger = logger.getChild('youtube-dl') + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + ydl.params['logger'] = ydl_logger + info = ydl.extract_info(url, download=False) + ydl.sanitize_info(info) + return info + + +@limit_concurrent(1) +@wrol_mode_check +async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT): + """ + Fetches Video info json for videos without comments, if comments are found then the info json file is replaced. + """ + one_month_ago = now() - timedelta(days=30) + + # Get any videos over a month old that do not have comments. + with get_db_session() as session: + videos = session.query(Video).filter( + # Have yet to get comments. + Video.have_comments == False, + Video.comments_failed == False, + # Need a URL to try to get comments. + FileGroup.url != None, # noqa + # We want old videos (time for comments to accumulate), or those which we don't know the published date. + or_( + one_month_ago > FileGroup.published_datetime, + FileGroup.published_datetime == None, + ), + ).join(FileGroup) \ + .order_by(FileGroup.published_datetime.nullsfirst()) \ + .limit(limit) + video_urls = [i.file_group.url for i in videos] + + logger.info(f'Found {len(video_urls)} videos missing comments') + + def add_video_to_skip_list(video_url: str): + """Do not attempt to get comments of the provided Video again.""" + with get_db_session(commit=True) as session_: + for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all(): + video_.comments_failed = True + + for url in video_urls: + # Sleep to catch cancel. + await asyncio.sleep(0) + + if download_manager.is_skipped(url): + # This video is skipped, do not try to get comments again. + add_video_to_skip_list(url) + + with get_db_session(commit=True) as session: + have_comments = False + for video in session.query(Video).join(FileGroup).filter_by(url=url).all(): + if (comments := video.get_comments()) and len(comments) >= 5: + # We already have enough comments. DB may have been wiped, we don't want to fetch comments again. + logger.info(f'Already have comments for Video: {url=}') + have_comments = video.have_comments = True + + if have_comments: + download_manager.add_to_skip_list(url) + continue + + tries = 5 + while tries > 0: + try: + tries -= 1 + # Get info json about the video. Retry until we have a list of comments. + info = download_video_info_json(url) + if info and isinstance(info.get('comments'), list): + break + except Exception as e: + logger.error(f'Got error when attempting to download comments: {url=}', exc_info=e) + else: + logger.error(f'Never got comments for video: {url=}') + download_manager.add_to_skip_list(url) + add_video_to_skip_list(url) + continue + + logger.debug(f'Got {len(info["comments"])} comments for Video: {url=}') + + with get_db_session(commit=True) as session: + for video in session.query(Video).join(FileGroup).filter_by(url=url).all(): + video.replace_info_json(info) + video.have_comments = True + + # Sleep a random amount of time, so we don't spam. + await asyncio.sleep(random.randint(2, 20)) diff --git a/modules/zim/conftest.py b/modules/zim/conftest.py index 493ad18b..5309cf78 100644 --- a/modules/zim/conftest.py +++ b/modules/zim/conftest.py @@ -85,7 +85,7 @@ def zim_path_factory(test_zim_bytes, test_directory): @pytest.fixture -def test_zim(test_session, zim_path_factory, test_directory) -> Zim: +def test_zim(test_session, zim_path_factory) -> Zim: zim = Zim.from_paths(test_session, zim_path_factory()) return zim diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 5fa2c441..55a102a5 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -19,7 +19,7 @@ from wrolpi.common import register_modeler, logger, extract_html_text, extract_h register_refresh_cleanup, background_task from wrolpi.db import get_db_session, optional_session, get_db_curs from wrolpi.downloader import DownloadFrequency -from wrolpi.files.lib import refresh_files +from wrolpi.files.lib import refresh_files, split_file_name_words from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST, DOCKERIZED @@ -62,6 +62,8 @@ async def zim_modeler(): session.add(zim) session.flush([zim]) zim_id = zim.id + file_group.title = file_group.primary_path.name + file_group.a_text = split_file_name_words(file_group.primary_path.name) file_group.indexed = True except Exception as e: if PYTEST: diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index cb7965e2..731fb66b 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -328,3 +328,16 @@ async def test_zim_subscription_download_import(test_session, test_downloader_co assert subscription.download_id == recurring.id assert subscription.name == 'Wikisource' assert subscription.language == 'en' + + +@pytest.mark.asyncio +async def test_zim_modeler(test_session, zim_factory): + """Zim modeler sets FileGroup title and a_text.""" + zim_factory('the_zim_title') + test_session.commit() + + await files_lib.refresh_files() + + fg: FileGroup = test_session.query(FileGroup).one() + assert fg.title == 'the_zim_title' + assert fg.a_text == 'the zim title' diff --git a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh index 044bdb6e..d44b4876 100755 --- a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh @@ -32,6 +32,14 @@ Type=Link URL=http://127.0.0.1:8086 Icon=/opt/wrolpi-help/venv/lib/python3.11/site-packages/mkdocs/themes/mkdocs/img/favicon.ico EOF +cat >/home/pi/Desktop/wrolpi-help-files.desktop <<'EOF' +[Desktop Entry] +Encoding=UTF-8 +Name=WROLPi Help Files +Type=Link +URL=/opt/wrolpi-help/docs/ +Icon=/usr/share/icons/gnome/48x48/places/folder.png +EOF chown -R pi:pi /home/pi chmod 0600 /home/pi/.pgpass usermod -aG wrolpi pi diff --git a/wrolpi/db.py b/wrolpi/db.py index 0b2bf015..12379e02 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -80,9 +80,6 @@ def get_db_context() -> Tuple[sqlalchemy.create_engine, Session]: return local_engine, session -# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) - - @contextmanager def get_db_session(commit: bool = False) -> ContextManager[Session]: """ diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 98a9cadd..c5789d3e 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -75,6 +75,8 @@ FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100)) FILE_MAX_PDF_SIZE = int(os.environ.get('FILE_MAX_PDF_SIZE', 40_000_000)) FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000)) +VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 50)) + DEFAULT_HTTP_HEADERS = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36', }