diff --git a/app/src/api.js b/app/src/api.js index b1ae5548..b407b351 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -172,6 +172,22 @@ export async function deleteVideos(videoIds) { } } +export async function downloadVideoMetadata(videoUrl, destination) { + return await postDownload( + [videoUrl], + 'video', + null, + null, + null, + destination, + null, + null, + null, + null, + true, + ); +} + export async function getDirectories(search_str) { let form_data = {'search_str': search_str || null}; let response = await apiPost(`${API_URI}/files/directories`, form_data); @@ -450,6 +466,7 @@ export async function postDownload( depth, suffix, max_pages, + doNotDownload, ) { if (!downloader) { toast({ @@ -477,7 +494,10 @@ export async function postDownload( if (tagNames) { body['tag_names'] = tagNames; } - let response = await apiPost(`${API_URI}/download`, body); + if (doNotDownload) { + body['do_not_download'] = doNotDownload; + } + const response = await apiPost(`${API_URI}/download`, body); return response; } diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 12b9c86c..0bea8ef8 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -22,7 +22,6 @@ import { useLoad, useSearchDirectories, useSearchOrder, - useSettings, useThrottle, useWROLMode } from "../hooks/customHooks"; @@ -1644,3 +1643,13 @@ export function useIsIgnoredDirectory(directory) { return ignoredDirectories.indexOf(directory) >= 0; } + +export function getParentDirectory(filePath) { + // Remove trailing slashes for consistency + const normalizedPath = filePath.endsWith('/') ? filePath.slice(0, -1) : filePath; + + // Find the last occurrence of "/" and extract the substring up to it + const parentDirectory = normalizedPath.substring(0, normalizedPath.lastIndexOf('/')); + + return parentDirectory; +} diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 4baabef1..c670e896 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -1,11 +1,12 @@ import React, {useContext} from 'react'; -import {deleteVideos, tagFileGroup, untagFileGroup} from "../api"; +import {deleteVideos, downloadVideoMetadata, tagFileGroup, untagFileGroup} from "../api"; import {Link, useNavigate, useParams} from "react-router-dom"; import _ from "lodash"; import { APIButton, BackButton, encodeMediaPath, + getParentDirectory, humanFileSize, humanNumber, isoDatetimeToString, @@ -21,7 +22,7 @@ import {ThemeContext} from "../contexts/contexts"; import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme"; import {VideoCard} from "./Videos"; import {TagsSelector} from "../Tags"; -import {Label} from "semantic-ui-react"; +import {Comment, CommentGroup, Label} from "semantic-ui-react"; const MEDIA_PATH = '/media'; @@ -67,6 +68,90 @@ export function CaptionTrack({src, ...props}) { return } +const NoComments = ({video}) => { + const videoUrl = video.url; + + const handleRefresh = async () => { + const destination = getParentDirectory(video.primary_path); + await downloadVideoMetadata(videoUrl, destination); + }; + + if (videoUrl) { + return +

No comments have been been downloaded. Refresh the video?

+ + Refresh +
+ } else { + return +

No comment have been downloaded. WROLPi does not know the URL of this video, so it cannot download + the comments.

+ + Try finding and downloading the video again. +
+ } +} + +const VideoComment = ({comment, children}) => { + const {t} = React.useContext(ThemeContext); + + const favoriteComment = ; + const likesCount = + {comment['like_count']} + ; + + return + + {comment['author']} + +
{comment['is_favorited'] && favoriteComment}
+
{isoDatetimeToString(comment['timestamp'])}
+
{comment['like_count'] && likesCount}
+
+ {comment['text']} +
+ {children && !_.isEmpty(children) && + + {children.map((i, key) => )} + + } +
+} + + +const Comments = ({comments, video}) => { + if (!comments || _.isEmpty(comments)) { + return + } + + comments.sort((a, b) => { + // Treat undefined like_count as 0 + const likeA = a.like_count ?? 0; + const likeB = b.like_count ?? 0; + + // For descending order + return likeB - likeA; + }); + + const rootComments = comments.filter(i => i['parent'] === 'root'); + + const sortedByParent = {}; + comments.forEach(obj => { + if (!sortedByParent[obj.parent]) { + sortedByParent[obj.parent] = []; + } + sortedByParent[obj.parent].push(obj); + }) + + return + {rootComments.map((i, key) => )} + +} + function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { const {theme} = useContext(ThemeContext); @@ -100,6 +185,11 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { navigate(-1); } + const handleRefresh = async () => { + const destination = getParentDirectory(videoFile.primary_path); + await downloadVideoMetadata(videoFile.url, destination); + }; + let videoUrl = `${MEDIA_PATH}/${encodeMediaPath(video.video_path)}`; let downloadUrl = `/download/${encodeMediaPath(video.video_path)}`; @@ -116,7 +206,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { description = formatVideoDescription(description, setVideoTime) } - let descriptionPane = { + const descriptionPane = { menuItem: 'Description', render: () =>
                 {description}
@@ -124,7 +214,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
         
     };
 
-    let aboutPane = {
+    const aboutPane = {
         menuItem: 'About', render: () => 
             

Size

{videoFile.size ? humanFileSize(videoFile.size) : 'Unknown'}

@@ -143,13 +233,13 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
} - let captionsPane = { + const captionsPane = { menuItem: 'Captions', render: () =>
{video.caption || 'No captions available.'}
}; - const {poster_file, info_json_file} = video; + const {poster_file, info_json_file, comments} = video; const filesPane = { menuItem: 'Files', render: () =>

Video File

@@ -181,7 +271,14 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
} - const tabPanes = [descriptionPane, aboutPane, filesPane, captionsPane]; + const commentsPane = { + menuItem: 'Comments', render: () => +
Top Comments
+ +
, + }; + + const tabPanes = [commentsPane, descriptionPane, aboutPane, filesPane, captionsPane]; const tabMenu = theme === darkTheme ? {inverted: true, attached: true} : {attached: true}; const localAddTag = async (name) => { @@ -260,6 +357,10 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { onClick={async () => await handleDeleteVideo(video.id)} obeyWROLMode={true} >Delete + Refresh


diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index ae49ce8b..0c4ac19e 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -2,6 +2,7 @@ import json import pathlib import re +import tempfile import traceback from abc import ABC from typing import Tuple, List, Dict @@ -19,7 +20,7 @@ from wrolpi.db import get_db_session from wrolpi.db import optional_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError -from wrolpi.files.lib import glob_shared_stem +from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel @@ -27,8 +28,8 @@ from .common import get_no_channel_directory, get_videos_directory, update_view_ from .errors import UnknownChannel from .lib import get_downloader_config from .models import Video, Channel -from .schema import ChannelPostRequest from .normalize_video_url import normalize_video_url +from .schema import ChannelPostRequest logger = logger.getChild(__name__) ydl_logger = logger.getChild('youtube-dl') @@ -283,6 +284,10 @@ class VideoDownloader(Downloader, ABC): logs = None # noqa try: + if settings.get('do_not_download'): + # User has requested refresh of video metadata, skip the rest of the video downloader. + return await self.download_info_json(download) + video_path, entry = self.prepare_filename(url, out_dir) # Do the real download. file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' @@ -303,6 +308,9 @@ class VideoDownloader(Downloader, ABC): '-o', file_name_format, '--no-cache-dir', '--compat-options', 'no-live-chat', + # Get top 20 comments. + '--write-comments', + '--extractor-args', 'youtube:max_comments=20,all,10;comment_sort=top', # Use experimental feature to merge files. '--ppa', 'Merger+ffmpeg_o1:-strict -2', url, @@ -438,6 +446,113 @@ class VideoDownloader(Downloader, ABC): logger.debug(f'Downloading {url} to {out_dir}') return final_filename, entry + async def download_info_json(self, download: Download) -> DownloadResult: + """User has requested to refresh Video metadata. Download only metadata files to temporary directory. + Replace existing metadata files only after successful download.""" + url = download.url + + # Download metadata files into temporary directory so yt-dlp always downloads the latest files. + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir = pathlib.Path(tmp_dir) + + # Download the metadata files, but skip the video file. + file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' + cmd = ( + str(YT_DLP_BIN), + '-f', PREFERRED_VIDEO_FORMAT, + '--skip-download', # SKIP THE VIDEO FILE. + '--match-filter', '!is_live', # Do not attempt to download Live videos. + '--write-info-json', + '--merge-output-format', PREFERRED_VIDEO_EXTENSION, + '--remux-video', PREFERRED_VIDEO_EXTENSION, + '-o', file_name_format, + '--no-cache-dir', + '--compat-options', 'no-live-chat', + # Get top 20 comments, 10 replies per parent. + '--write-comments', + '--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top', + url, + ) + return_code, logs, _ = await self.process_runner(url, cmd, tmp_dir) + + stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr'] + stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout'] + + if return_code != 0: + error = f'{stdout}\n\n\n{stderr}\n\nvideo info json downloader process exited with {return_code}' + return DownloadResult( + success=False, + error=error, + ) + + # Find the info json file in the temporary directory. + info_json_paths = [i for i in tmp_dir.iterdir() if i.name.endswith('.info.json') and i.stat().st_size] + if len(info_json_paths) == 0: + return DownloadResult( + success=False, + error='No info json file was downloaded.' + ) + elif len(info_json_paths) > 1: + return DownloadResult( + success=False, + error='More than one info json file was downloaded.' + ) + + info_json_path = info_json_paths[0] + + if not info_json_path.stat().st_size: + return DownloadResult( + success=False, + error='Video info json file was empty', + ) + + # Reformat new info json file for human reading. + format_json_file(info_json_path) + + with get_db_session(commit=True) as session: + # Find the existing video, replace its info json. + video = Video.get_by_url(url, session) + video_id = video.id + channel_id = video.channel_id + + if video.info_json_path and video.info_json_path.is_file(): + # Move old file temporarily. Only delete old data if the swap is successful. + logger.debug(f'Replacing info json file: {video.info_json_path}') + tmp_info_json_path = pathlib.Path(f'{video.info_json_path}.tmp') + if tmp_info_json_path.is_file(): + if video.info_json_path.stat().st_size: + tmp_info_json_path.unlink() + else: + raise UnrecoverableDownloadError( + f'Temporary info json file already exists: {tmp_info_json_path}') + + # Move existing info json file to a temporary location. Move it back if this fails. + video.info_json_path.rename(tmp_info_json_path) + try: + video.info_json_path.write_text(info_json_path.read_text()) + tmp_info_json_path.unlink() + except Exception as e: + logger.error('Failed to swap video info json file', exc_info=e) + tmp_info_json_path.rename(video.info_json_path) + raise + else: + # info json file does not exist, write one next to the video. + video_path = video.video_path + stem, _ = split_path_stem_and_suffix(video_path) + new_info_json_path = pathlib.Path(f'{stem}.info.json') + new_info_json_path.write_text(info_json_path.read_text()) + video.file_group.append_files(new_info_json_path) + + if channel_id: + location = f'/videos/channel/{channel_id}/video/{video_id}' + else: + location = f'/videos/video/{video_id}' + + logger.debug(f'Downloaded video info json {location=}') + logger.info(f'Successfully downloaded video info json {url} {video}') + + return DownloadResult(success=True, location=location) + channel_downloader = ChannelDownloader() # Videos may match the ChannelDownloader, give it a higher priority. diff --git a/modules/videos/models.py b/modules/videos/models.py index 8bdbfc61..370bc0a9 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -60,6 +60,8 @@ class Video(ModelHelper, Base): except Exception as e: logger.error(f'{self} ffprobe_json is invalid', exc_info=e) + info_json = self.get_info_json() or dict() + # Put live data in "video" instead of "data" to avoid confusion on the frontend. d['video'] = dict( caption=self.file_group.d_text, @@ -68,6 +70,7 @@ class Video(ModelHelper, Base): channel_id=self.channel_id, codec_names=codec_names, codec_types=codec_types, + comments=info_json.get('comments'), description=self.file_group.c_text or self.get_video_description(), id=self.id, info_json_file=self.info_json_file, @@ -304,6 +307,14 @@ class Video(ModelHelper, Base): .filter(FileGroup.primary_path == path).one_or_none() return video + @staticmethod + @optional_session + def get_by_url(url: str, session: Session) -> Optional['Video']: + video = session.query(Video) \ + .join(FileGroup, FileGroup.id == Video.file_group_id) \ + .filter(FileGroup.url == url).one_or_none() + return video + @staticmethod @optional_session def get_by_id(id_: int, session: Session = None) -> Optional['Video']: diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 84747c90..94b4daee 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -244,10 +244,9 @@ async def post_download(_: Request, body: schema.DownloadRequest): # Don't overwrite settings if restarting download. settings = None - if excluded_urls or destination or body.tag_names or body.suffix or body.depth: + if excluded_urls or destination or body.tag_names or body.suffix or body.depth or body.do_not_download: settings = dict(excluded_urls=excluded_urls, destination=destination, tag_names=body.tag_names, - suffix=body.suffix, - depth=body.depth) + suffix=body.suffix, do_not_download=body.do_not_download, depth=body.depth) if body.frequency: download_manager.recurring_download(body.urls[0], body.frequency, downloader_name=body.downloader, diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 60a7e728..1a6ff6f8 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -62,6 +62,7 @@ class DownloadRequest: suffix: Optional[str] = None depth: Optional[int] = None max_pages: Optional[int] = None + do_not_download: Optional[bool] = False def __post_init__(self): urls = [j for i in self.urls if (j := i.strip())]