diff --git a/app/src/api.js b/app/src/api.js
index b1ae5548..b407b351 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -172,6 +172,22 @@ export async function deleteVideos(videoIds) {
}
}
+export async function downloadVideoMetadata(videoUrl, destination) {
+ return await postDownload(
+ [videoUrl],
+ 'video',
+ null,
+ null,
+ null,
+ destination,
+ null,
+ null,
+ null,
+ null,
+ true,
+ );
+}
+
export async function getDirectories(search_str) {
let form_data = {'search_str': search_str || null};
let response = await apiPost(`${API_URI}/files/directories`, form_data);
@@ -450,6 +466,7 @@ export async function postDownload(
depth,
suffix,
max_pages,
+ doNotDownload,
) {
if (!downloader) {
toast({
@@ -477,7 +494,10 @@ export async function postDownload(
if (tagNames) {
body['tag_names'] = tagNames;
}
- let response = await apiPost(`${API_URI}/download`, body);
+ if (doNotDownload) {
+ body['do_not_download'] = doNotDownload;
+ }
+ const response = await apiPost(`${API_URI}/download`, body);
return response;
}
diff --git a/app/src/components/Common.js b/app/src/components/Common.js
index 12b9c86c..0bea8ef8 100644
--- a/app/src/components/Common.js
+++ b/app/src/components/Common.js
@@ -22,7 +22,6 @@ import {
useLoad,
useSearchDirectories,
useSearchOrder,
- useSettings,
useThrottle,
useWROLMode
} from "../hooks/customHooks";
@@ -1644,3 +1643,13 @@ export function useIsIgnoredDirectory(directory) {
return ignoredDirectories.indexOf(directory) >= 0;
}
+
+export function getParentDirectory(filePath) {
+ // Remove trailing slashes for consistency
+ const normalizedPath = filePath.endsWith('/') ? filePath.slice(0, -1) : filePath;
+
+ // Find the last occurrence of "/" and extract the substring up to it
+ const parentDirectory = normalizedPath.substring(0, normalizedPath.lastIndexOf('/'));
+
+ return parentDirectory;
+}
diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js
index 4baabef1..c670e896 100644
--- a/app/src/components/VideoPlayer.js
+++ b/app/src/components/VideoPlayer.js
@@ -1,11 +1,12 @@
import React, {useContext} from 'react';
-import {deleteVideos, tagFileGroup, untagFileGroup} from "../api";
+import {deleteVideos, downloadVideoMetadata, tagFileGroup, untagFileGroup} from "../api";
import {Link, useNavigate, useParams} from "react-router-dom";
import _ from "lodash";
import {
APIButton,
BackButton,
encodeMediaPath,
+ getParentDirectory,
humanFileSize,
humanNumber,
isoDatetimeToString,
@@ -21,7 +22,7 @@ import {ThemeContext} from "../contexts/contexts";
import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme";
import {VideoCard} from "./Videos";
import {TagsSelector} from "../Tags";
-import {Label} from "semantic-ui-react";
+import {Comment, CommentGroup, Label} from "semantic-ui-react";
const MEDIA_PATH = '/media';
@@ -67,6 +68,90 @@ export function CaptionTrack({src, ...props}) {
return
}
+const NoComments = ({video}) => {
+ const videoUrl = video.url;
+
+ const handleRefresh = async () => {
+ const destination = getParentDirectory(video.primary_path);
+ await downloadVideoMetadata(videoUrl, destination);
+ };
+
+ if (videoUrl) {
+ return
+ No comments have been been downloaded. Refresh the video?
+
+ Refresh
+
+ } else {
+ return
+ No comment have been downloaded. WROLPi does not know the URL of this video, so it cannot download
+ the comments.
+
+ Try finding and downloading the video again.
+
+ }
+}
+
+const VideoComment = ({comment, children}) => {
+ const {t} = React.useContext(ThemeContext);
+
+ const favoriteComment = ;
+ const likesCount =
+ {comment['like_count']}
+ ;
+
+ return
+
+ {comment['author']}
+
+ {comment['is_favorited'] && favoriteComment}
+ {isoDatetimeToString(comment['timestamp'])}
+ {comment['like_count'] && likesCount}
+
+ {comment['text']}
+
+ {children && !_.isEmpty(children) &&
+
+ {children.map((i, key) => )}
+
+ }
+
+}
+
+
+const Comments = ({comments, video}) => {
+ if (!comments || _.isEmpty(comments)) {
+ return
+ }
+
+ comments.sort((a, b) => {
+ // Treat undefined like_count as 0
+ const likeA = a.like_count ?? 0;
+ const likeB = b.like_count ?? 0;
+
+ // For descending order
+ return likeB - likeA;
+ });
+
+ const rootComments = comments.filter(i => i['parent'] === 'root');
+
+ const sortedByParent = {};
+ comments.forEach(obj => {
+ if (!sortedByParent[obj.parent]) {
+ sortedByParent[obj.parent] = [];
+ }
+ sortedByParent[obj.parent].push(obj);
+ })
+
+ return
+ {rootComments.map((i, key) => )}
+
+}
+
function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
const {theme} = useContext(ThemeContext);
@@ -100,6 +185,11 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
navigate(-1);
}
+ const handleRefresh = async () => {
+ const destination = getParentDirectory(videoFile.primary_path);
+ await downloadVideoMetadata(videoFile.url, destination);
+ };
+
let videoUrl = `${MEDIA_PATH}/${encodeMediaPath(video.video_path)}`;
let downloadUrl = `/download/${encodeMediaPath(video.video_path)}`;
@@ -116,7 +206,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
description = formatVideoDescription(description, setVideoTime)
}
- let descriptionPane = {
+ const descriptionPane = {
menuItem: 'Description', render: () =>
{description}
@@ -124,7 +214,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
};
- let aboutPane = {
+ const aboutPane = {
menuItem: 'About', render: () =>
Size
{videoFile.size ? humanFileSize(videoFile.size) : 'Unknown'}
@@ -143,13 +233,13 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
}
- let captionsPane = {
+ const captionsPane = {
menuItem: 'Captions', render: () =>
{video.caption || 'No captions available.'}
};
- const {poster_file, info_json_file} = video;
+ const {poster_file, info_json_file, comments} = video;
const filesPane = {
menuItem: 'Files', render: () =>
Video File
@@ -181,7 +271,14 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
}
- const tabPanes = [descriptionPane, aboutPane, filesPane, captionsPane];
+ const commentsPane = {
+ menuItem: 'Comments', render: () =>
+
+
+ ,
+ };
+
+ const tabPanes = [commentsPane, descriptionPane, aboutPane, filesPane, captionsPane];
const tabMenu = theme === darkTheme ? {inverted: true, attached: true} : {attached: true};
const localAddTag = async (name) => {
@@ -260,6 +357,10 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
onClick={async () => await handleDeleteVideo(video.id)}
obeyWROLMode={true}
>Delete
+ Refresh
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index ae49ce8b..0c4ac19e 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -2,6 +2,7 @@
import json
import pathlib
import re
+import tempfile
import traceback
from abc import ABC
from typing import Tuple, List, Dict
@@ -19,7 +20,7 @@ from wrolpi.db import get_db_session
from wrolpi.db import optional_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
-from wrolpi.files.lib import glob_shared_stem
+from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
@@ -27,8 +28,8 @@ from .common import get_no_channel_directory, get_videos_directory, update_view_
from .errors import UnknownChannel
from .lib import get_downloader_config
from .models import Video, Channel
-from .schema import ChannelPostRequest
from .normalize_video_url import normalize_video_url
+from .schema import ChannelPostRequest
logger = logger.getChild(__name__)
ydl_logger = logger.getChild('youtube-dl')
@@ -283,6 +284,10 @@ class VideoDownloader(Downloader, ABC):
logs = None # noqa
try:
+ if settings.get('do_not_download'):
+ # User has requested refresh of video metadata, skip the rest of the video downloader.
+ return await self.download_info_json(download)
+
video_path, entry = self.prepare_filename(url, out_dir)
# Do the real download.
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
@@ -303,6 +308,9 @@ class VideoDownloader(Downloader, ABC):
'-o', file_name_format,
'--no-cache-dir',
'--compat-options', 'no-live-chat',
+ # Get top 20 comments.
+ '--write-comments',
+ '--extractor-args', 'youtube:max_comments=20,all,10;comment_sort=top',
# Use experimental feature to merge files.
'--ppa', 'Merger+ffmpeg_o1:-strict -2',
url,
@@ -438,6 +446,113 @@ class VideoDownloader(Downloader, ABC):
logger.debug(f'Downloading {url} to {out_dir}')
return final_filename, entry
+ async def download_info_json(self, download: Download) -> DownloadResult:
+ """User has requested to refresh Video metadata. Download only metadata files to temporary directory.
+ Replace existing metadata files only after successful download."""
+ url = download.url
+
+ # Download metadata files into temporary directory so yt-dlp always downloads the latest files.
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_dir = pathlib.Path(tmp_dir)
+
+ # Download the metadata files, but skip the video file.
+ file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
+ cmd = (
+ str(YT_DLP_BIN),
+ '-f', PREFERRED_VIDEO_FORMAT,
+ '--skip-download', # SKIP THE VIDEO FILE.
+ '--match-filter', '!is_live', # Do not attempt to download Live videos.
+ '--write-info-json',
+ '--merge-output-format', PREFERRED_VIDEO_EXTENSION,
+ '--remux-video', PREFERRED_VIDEO_EXTENSION,
+ '-o', file_name_format,
+ '--no-cache-dir',
+ '--compat-options', 'no-live-chat',
+ # Get top 20 comments, 10 replies per parent.
+ '--write-comments',
+ '--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
+ url,
+ )
+ return_code, logs, _ = await self.process_runner(url, cmd, tmp_dir)
+
+ stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
+ stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
+
+ if return_code != 0:
+ error = f'{stdout}\n\n\n{stderr}\n\nvideo info json downloader process exited with {return_code}'
+ return DownloadResult(
+ success=False,
+ error=error,
+ )
+
+ # Find the info json file in the temporary directory.
+ info_json_paths = [i for i in tmp_dir.iterdir() if i.name.endswith('.info.json') and i.stat().st_size]
+ if len(info_json_paths) == 0:
+ return DownloadResult(
+ success=False,
+ error='No info json file was downloaded.'
+ )
+ elif len(info_json_paths) > 1:
+ return DownloadResult(
+ success=False,
+ error='More than one info json file was downloaded.'
+ )
+
+ info_json_path = info_json_paths[0]
+
+ if not info_json_path.stat().st_size:
+ return DownloadResult(
+ success=False,
+ error='Video info json file was empty',
+ )
+
+ # Reformat new info json file for human reading.
+ format_json_file(info_json_path)
+
+ with get_db_session(commit=True) as session:
+ # Find the existing video, replace its info json.
+ video = Video.get_by_url(url, session)
+ video_id = video.id
+ channel_id = video.channel_id
+
+ if video.info_json_path and video.info_json_path.is_file():
+ # Move old file temporarily. Only delete old data if the swap is successful.
+ logger.debug(f'Replacing info json file: {video.info_json_path}')
+ tmp_info_json_path = pathlib.Path(f'{video.info_json_path}.tmp')
+ if tmp_info_json_path.is_file():
+ if video.info_json_path.stat().st_size:
+ tmp_info_json_path.unlink()
+ else:
+ raise UnrecoverableDownloadError(
+ f'Temporary info json file already exists: {tmp_info_json_path}')
+
+ # Move existing info json file to a temporary location. Move it back if this fails.
+ video.info_json_path.rename(tmp_info_json_path)
+ try:
+ video.info_json_path.write_text(info_json_path.read_text())
+ tmp_info_json_path.unlink()
+ except Exception as e:
+ logger.error('Failed to swap video info json file', exc_info=e)
+ tmp_info_json_path.rename(video.info_json_path)
+ raise
+ else:
+ # info json file does not exist, write one next to the video.
+ video_path = video.video_path
+ stem, _ = split_path_stem_and_suffix(video_path)
+ new_info_json_path = pathlib.Path(f'{stem}.info.json')
+ new_info_json_path.write_text(info_json_path.read_text())
+ video.file_group.append_files(new_info_json_path)
+
+ if channel_id:
+ location = f'/videos/channel/{channel_id}/video/{video_id}'
+ else:
+ location = f'/videos/video/{video_id}'
+
+ logger.debug(f'Downloaded video info json {location=}')
+ logger.info(f'Successfully downloaded video info json {url} {video}')
+
+ return DownloadResult(success=True, location=location)
+
channel_downloader = ChannelDownloader()
# Videos may match the ChannelDownloader, give it a higher priority.
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 8bdbfc61..370bc0a9 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -60,6 +60,8 @@ class Video(ModelHelper, Base):
except Exception as e:
logger.error(f'{self} ffprobe_json is invalid', exc_info=e)
+ info_json = self.get_info_json() or dict()
+
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
d['video'] = dict(
caption=self.file_group.d_text,
@@ -68,6 +70,7 @@ class Video(ModelHelper, Base):
channel_id=self.channel_id,
codec_names=codec_names,
codec_types=codec_types,
+ comments=info_json.get('comments'),
description=self.file_group.c_text or self.get_video_description(),
id=self.id,
info_json_file=self.info_json_file,
@@ -304,6 +307,14 @@ class Video(ModelHelper, Base):
.filter(FileGroup.primary_path == path).one_or_none()
return video
+ @staticmethod
+ @optional_session
+ def get_by_url(url: str, session: Session) -> Optional['Video']:
+ video = session.query(Video) \
+ .join(FileGroup, FileGroup.id == Video.file_group_id) \
+ .filter(FileGroup.url == url).one_or_none()
+ return video
+
@staticmethod
@optional_session
def get_by_id(id_: int, session: Session = None) -> Optional['Video']:
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 84747c90..94b4daee 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -244,10 +244,9 @@ async def post_download(_: Request, body: schema.DownloadRequest):
# Don't overwrite settings if restarting download.
settings = None
- if excluded_urls or destination or body.tag_names or body.suffix or body.depth:
+ if excluded_urls or destination or body.tag_names or body.suffix or body.depth or body.do_not_download:
settings = dict(excluded_urls=excluded_urls, destination=destination, tag_names=body.tag_names,
- suffix=body.suffix,
- depth=body.depth)
+ suffix=body.suffix, do_not_download=body.do_not_download, depth=body.depth)
if body.frequency:
download_manager.recurring_download(body.urls[0], body.frequency, downloader_name=body.downloader,
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index 60a7e728..1a6ff6f8 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -62,6 +62,7 @@ class DownloadRequest:
suffix: Optional[str] = None
depth: Optional[int] = None
max_pages: Optional[int] = None
+ do_not_download: Optional[bool] = False
def __post_init__(self):
urls = [j for i in self.urls if (j := i.strip())]