mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Videos display comments. User can download comments using Refresh button.
This commit is contained in:
Roland
parent
ea4d640100
commit
f865d9fb0f
7 changed files with 270 additions and 14 deletions
|
|
@ -172,6 +172,22 @@ export async function deleteVideos(videoIds) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function downloadVideoMetadata(videoUrl, destination) {
|
||||
return await postDownload(
|
||||
[videoUrl],
|
||||
'video',
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
destination,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
export async function getDirectories(search_str) {
|
||||
let form_data = {'search_str': search_str || null};
|
||||
let response = await apiPost(`${API_URI}/files/directories`, form_data);
|
||||
|
|
@ -450,6 +466,7 @@ export async function postDownload(
|
|||
depth,
|
||||
suffix,
|
||||
max_pages,
|
||||
doNotDownload,
|
||||
) {
|
||||
if (!downloader) {
|
||||
toast({
|
||||
|
|
@ -477,7 +494,10 @@ export async function postDownload(
|
|||
if (tagNames) {
|
||||
body['tag_names'] = tagNames;
|
||||
}
|
||||
let response = await apiPost(`${API_URI}/download`, body);
|
||||
if (doNotDownload) {
|
||||
body['do_not_download'] = doNotDownload;
|
||||
}
|
||||
const response = await apiPost(`${API_URI}/download`, body);
|
||||
return response;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import {
|
|||
useLoad,
|
||||
useSearchDirectories,
|
||||
useSearchOrder,
|
||||
useSettings,
|
||||
useThrottle,
|
||||
useWROLMode
|
||||
} from "../hooks/customHooks";
|
||||
|
|
@ -1644,3 +1643,13 @@ export function useIsIgnoredDirectory(directory) {
|
|||
|
||||
return ignoredDirectories.indexOf(directory) >= 0;
|
||||
}
|
||||
|
||||
export function getParentDirectory(filePath) {
|
||||
// Remove trailing slashes for consistency
|
||||
const normalizedPath = filePath.endsWith('/') ? filePath.slice(0, -1) : filePath;
|
||||
|
||||
// Find the last occurrence of "/" and extract the substring up to it
|
||||
const parentDirectory = normalizedPath.substring(0, normalizedPath.lastIndexOf('/'));
|
||||
|
||||
return parentDirectory;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import React, {useContext} from 'react';
|
||||
import {deleteVideos, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {deleteVideos, downloadVideoMetadata, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {Link, useNavigate, useParams} from "react-router-dom";
|
||||
import _ from "lodash";
|
||||
import {
|
||||
APIButton,
|
||||
BackButton,
|
||||
encodeMediaPath,
|
||||
getParentDirectory,
|
||||
humanFileSize,
|
||||
humanNumber,
|
||||
isoDatetimeToString,
|
||||
|
|
@ -21,7 +22,7 @@ import {ThemeContext} from "../contexts/contexts";
|
|||
import {Button, darkTheme, Header, Icon, Segment, Tab, TabPane} from "./Theme";
|
||||
import {VideoCard} from "./Videos";
|
||||
import {TagsSelector} from "../Tags";
|
||||
import {Label} from "semantic-ui-react";
|
||||
import {Comment, CommentGroup, Label} from "semantic-ui-react";
|
||||
|
||||
const MEDIA_PATH = '/media';
|
||||
|
||||
|
|
@ -67,6 +68,90 @@ export function CaptionTrack({src, ...props}) {
|
|||
return <track kind="captions" label="English" src={src} srcLang="en" default {...props}/>
|
||||
}
|
||||
|
||||
const NoComments = ({video}) => {
|
||||
const videoUrl = video.url;
|
||||
|
||||
const handleRefresh = async () => {
|
||||
const destination = getParentDirectory(video.primary_path);
|
||||
await downloadVideoMetadata(videoUrl, destination);
|
||||
};
|
||||
|
||||
if (videoUrl) {
|
||||
return <React.Fragment>
|
||||
<p>No comments have been been downloaded. Refresh the video?</p>
|
||||
|
||||
<APIButton
|
||||
color='blue'
|
||||
size='large'
|
||||
onClick={handleRefresh}
|
||||
>Refresh</APIButton>
|
||||
</React.Fragment>
|
||||
} else {
|
||||
return <React.Fragment>
|
||||
<p>No comment have been downloaded. WROLPi does not know the URL of this video, so it cannot download
|
||||
the comments.</p>
|
||||
|
||||
<big>Try finding and downloading the video again.</big>
|
||||
</React.Fragment>
|
||||
}
|
||||
}
|
||||
|
||||
const VideoComment = ({comment, children}) => {
|
||||
const {t} = React.useContext(ThemeContext);
|
||||
|
||||
const favoriteComment = <Icon name='heart'/>;
|
||||
const likesCount = <React.Fragment>
|
||||
<Icon name='thumbs up'/> {comment['like_count']}
|
||||
</React.Fragment>;
|
||||
|
||||
return <Comment>
|
||||
<Comment.Content>
|
||||
<Comment.Author as='a'><span {...t}>{comment['author']}</span></Comment.Author>
|
||||
<Comment.Metadata>
|
||||
<div {...t}>{comment['is_favorited'] && favoriteComment}</div>
|
||||
<div {...t}>{isoDatetimeToString(comment['timestamp'])}</div>
|
||||
<div {...t}>{comment['like_count'] && likesCount}</div>
|
||||
</Comment.Metadata>
|
||||
<Comment.Text {...t}>{comment['text']}</Comment.Text>
|
||||
</Comment.Content>
|
||||
{children && !_.isEmpty(children) &&
|
||||
<CommentGroup>
|
||||
{children.map((i, key) => <VideoComment key={key} comment={i}/>)}
|
||||
</CommentGroup>
|
||||
}
|
||||
</Comment>
|
||||
}
|
||||
|
||||
|
||||
const Comments = ({comments, video}) => {
|
||||
if (!comments || _.isEmpty(comments)) {
|
||||
return <NoComments video={video}/>
|
||||
}
|
||||
|
||||
comments.sort((a, b) => {
|
||||
// Treat undefined like_count as 0
|
||||
const likeA = a.like_count ?? 0;
|
||||
const likeB = b.like_count ?? 0;
|
||||
|
||||
// For descending order
|
||||
return likeB - likeA;
|
||||
});
|
||||
|
||||
const rootComments = comments.filter(i => i['parent'] === 'root');
|
||||
|
||||
const sortedByParent = {};
|
||||
comments.forEach(obj => {
|
||||
if (!sortedByParent[obj.parent]) {
|
||||
sortedByParent[obj.parent] = [];
|
||||
}
|
||||
sortedByParent[obj.parent].push(obj);
|
||||
})
|
||||
|
||||
return <CommentGroup>
|
||||
{rootComments.map((i, key) => <VideoComment key={key} comment={i} children={sortedByParent[i.id]}/>)}
|
||||
</CommentGroup>
|
||||
}
|
||||
|
||||
function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
||||
const {theme} = useContext(ThemeContext);
|
||||
|
||||
|
|
@ -100,6 +185,11 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
navigate(-1);
|
||||
}
|
||||
|
||||
const handleRefresh = async () => {
|
||||
const destination = getParentDirectory(videoFile.primary_path);
|
||||
await downloadVideoMetadata(videoFile.url, destination);
|
||||
};
|
||||
|
||||
let videoUrl = `${MEDIA_PATH}/${encodeMediaPath(video.video_path)}`;
|
||||
let downloadUrl = `/download/${encodeMediaPath(video.video_path)}`;
|
||||
|
||||
|
|
@ -116,7 +206,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
description = formatVideoDescription(description, setVideoTime)
|
||||
}
|
||||
|
||||
let descriptionPane = {
|
||||
const descriptionPane = {
|
||||
menuItem: 'Description', render: () => <TabPane>
|
||||
<pre className="wrap-text">
|
||||
{description}
|
||||
|
|
@ -124,7 +214,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
</TabPane>
|
||||
};
|
||||
|
||||
let aboutPane = {
|
||||
const aboutPane = {
|
||||
menuItem: 'About', render: () => <TabPane>
|
||||
<h3>Size</h3>
|
||||
<p>{videoFile.size ? humanFileSize(videoFile.size) : 'Unknown'}</p>
|
||||
|
|
@ -143,13 +233,13 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
</TabPane>
|
||||
}
|
||||
|
||||
let captionsPane = {
|
||||
const captionsPane = {
|
||||
menuItem: 'Captions', render: () => <TabPane>
|
||||
<pre>{video.caption || 'No captions available.'}</pre>
|
||||
</TabPane>
|
||||
};
|
||||
|
||||
const {poster_file, info_json_file} = video;
|
||||
const {poster_file, info_json_file, comments} = video;
|
||||
const filesPane = {
|
||||
menuItem: 'Files', render: () => <TabPane>
|
||||
<h3>Video File</h3>
|
||||
|
|
@ -181,7 +271,14 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
</TabPane>
|
||||
}
|
||||
|
||||
const tabPanes = [descriptionPane, aboutPane, filesPane, captionsPane];
|
||||
const commentsPane = {
|
||||
menuItem: 'Comments', render: () => <TabPane>
|
||||
<Header as='h3'>Top Comments</Header>
|
||||
<Comments comments={comments} video={videoFile}/>
|
||||
</TabPane>,
|
||||
};
|
||||
|
||||
const tabPanes = [commentsPane, descriptionPane, aboutPane, filesPane, captionsPane];
|
||||
const tabMenu = theme === darkTheme ? {inverted: true, attached: true} : {attached: true};
|
||||
|
||||
const localAddTag = async (name) => {
|
||||
|
|
@ -260,6 +357,10 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
onClick={async () => await handleDeleteVideo(video.id)}
|
||||
obeyWROLMode={true}
|
||||
>Delete</APIButton>
|
||||
<APIButton
|
||||
color='blue'
|
||||
onClick={handleRefresh}
|
||||
>Refresh</APIButton>
|
||||
</p>
|
||||
<br/>
|
||||
</Segment>
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import tempfile
|
||||
import traceback
|
||||
from abc import ABC
|
||||
from typing import Tuple, List, Dict
|
||||
|
|
@ -19,7 +20,7 @@ from wrolpi.db import get_db_session
|
|||
from wrolpi.db import optional_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
|
|
@ -27,8 +28,8 @@ from .common import get_no_channel_directory, get_videos_directory, update_view_
|
|||
from .errors import UnknownChannel
|
||||
from .lib import get_downloader_config
|
||||
from .models import Video, Channel
|
||||
from .schema import ChannelPostRequest
|
||||
from .normalize_video_url import normalize_video_url
|
||||
from .schema import ChannelPostRequest
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
ydl_logger = logger.getChild('youtube-dl')
|
||||
|
|
@ -283,6 +284,10 @@ class VideoDownloader(Downloader, ABC):
|
|||
|
||||
logs = None # noqa
|
||||
try:
|
||||
if settings.get('do_not_download'):
|
||||
# User has requested refresh of video metadata, skip the rest of the video downloader.
|
||||
return await self.download_info_json(download)
|
||||
|
||||
video_path, entry = self.prepare_filename(url, out_dir)
|
||||
# Do the real download.
|
||||
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
|
||||
|
|
@ -303,6 +308,9 @@ class VideoDownloader(Downloader, ABC):
|
|||
'-o', file_name_format,
|
||||
'--no-cache-dir',
|
||||
'--compat-options', 'no-live-chat',
|
||||
# Get top 20 comments.
|
||||
'--write-comments',
|
||||
'--extractor-args', 'youtube:max_comments=20,all,10;comment_sort=top',
|
||||
# Use experimental feature to merge files.
|
||||
'--ppa', 'Merger+ffmpeg_o1:-strict -2',
|
||||
url,
|
||||
|
|
@ -438,6 +446,113 @@ class VideoDownloader(Downloader, ABC):
|
|||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
return final_filename, entry
|
||||
|
||||
async def download_info_json(self, download: Download) -> DownloadResult:
|
||||
"""User has requested to refresh Video metadata. Download only metadata files to temporary directory.
|
||||
Replace existing metadata files only after successful download."""
|
||||
url = download.url
|
||||
|
||||
# Download metadata files into temporary directory so yt-dlp always downloads the latest files.
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp_dir = pathlib.Path(tmp_dir)
|
||||
|
||||
# Download the metadata files, but skip the video file.
|
||||
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
|
||||
cmd = (
|
||||
str(YT_DLP_BIN),
|
||||
'-f', PREFERRED_VIDEO_FORMAT,
|
||||
'--skip-download', # SKIP THE VIDEO FILE.
|
||||
'--match-filter', '!is_live', # Do not attempt to download Live videos.
|
||||
'--write-info-json',
|
||||
'--merge-output-format', PREFERRED_VIDEO_EXTENSION,
|
||||
'--remux-video', PREFERRED_VIDEO_EXTENSION,
|
||||
'-o', file_name_format,
|
||||
'--no-cache-dir',
|
||||
'--compat-options', 'no-live-chat',
|
||||
# Get top 20 comments, 10 replies per parent.
|
||||
'--write-comments',
|
||||
'--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
|
||||
url,
|
||||
)
|
||||
return_code, logs, _ = await self.process_runner(url, cmd, tmp_dir)
|
||||
|
||||
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
|
||||
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
|
||||
|
||||
if return_code != 0:
|
||||
error = f'{stdout}\n\n\n{stderr}\n\nvideo info json downloader process exited with {return_code}'
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error=error,
|
||||
)
|
||||
|
||||
# Find the info json file in the temporary directory.
|
||||
info_json_paths = [i for i in tmp_dir.iterdir() if i.name.endswith('.info.json') and i.stat().st_size]
|
||||
if len(info_json_paths) == 0:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='No info json file was downloaded.'
|
||||
)
|
||||
elif len(info_json_paths) > 1:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='More than one info json file was downloaded.'
|
||||
)
|
||||
|
||||
info_json_path = info_json_paths[0]
|
||||
|
||||
if not info_json_path.stat().st_size:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Video info json file was empty',
|
||||
)
|
||||
|
||||
# Reformat new info json file for human reading.
|
||||
format_json_file(info_json_path)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Find the existing video, replace its info json.
|
||||
video = Video.get_by_url(url, session)
|
||||
video_id = video.id
|
||||
channel_id = video.channel_id
|
||||
|
||||
if video.info_json_path and video.info_json_path.is_file():
|
||||
# Move old file temporarily. Only delete old data if the swap is successful.
|
||||
logger.debug(f'Replacing info json file: {video.info_json_path}')
|
||||
tmp_info_json_path = pathlib.Path(f'{video.info_json_path}.tmp')
|
||||
if tmp_info_json_path.is_file():
|
||||
if video.info_json_path.stat().st_size:
|
||||
tmp_info_json_path.unlink()
|
||||
else:
|
||||
raise UnrecoverableDownloadError(
|
||||
f'Temporary info json file already exists: {tmp_info_json_path}')
|
||||
|
||||
# Move existing info json file to a temporary location. Move it back if this fails.
|
||||
video.info_json_path.rename(tmp_info_json_path)
|
||||
try:
|
||||
video.info_json_path.write_text(info_json_path.read_text())
|
||||
tmp_info_json_path.unlink()
|
||||
except Exception as e:
|
||||
logger.error('Failed to swap video info json file', exc_info=e)
|
||||
tmp_info_json_path.rename(video.info_json_path)
|
||||
raise
|
||||
else:
|
||||
# info json file does not exist, write one next to the video.
|
||||
video_path = video.video_path
|
||||
stem, _ = split_path_stem_and_suffix(video_path)
|
||||
new_info_json_path = pathlib.Path(f'{stem}.info.json')
|
||||
new_info_json_path.write_text(info_json_path.read_text())
|
||||
video.file_group.append_files(new_info_json_path)
|
||||
|
||||
if channel_id:
|
||||
location = f'/videos/channel/{channel_id}/video/{video_id}'
|
||||
else:
|
||||
location = f'/videos/video/{video_id}'
|
||||
|
||||
logger.debug(f'Downloaded video info json {location=}')
|
||||
logger.info(f'Successfully downloaded video info json {url} {video}')
|
||||
|
||||
return DownloadResult(success=True, location=location)
|
||||
|
||||
|
||||
channel_downloader = ChannelDownloader()
|
||||
# Videos may match the ChannelDownloader, give it a higher priority.
|
||||
|
|
|
|||
|
|
@ -60,6 +60,8 @@ class Video(ModelHelper, Base):
|
|||
except Exception as e:
|
||||
logger.error(f'{self} ffprobe_json is invalid', exc_info=e)
|
||||
|
||||
info_json = self.get_info_json() or dict()
|
||||
|
||||
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
|
||||
d['video'] = dict(
|
||||
caption=self.file_group.d_text,
|
||||
|
|
@ -68,6 +70,7 @@ class Video(ModelHelper, Base):
|
|||
channel_id=self.channel_id,
|
||||
codec_names=codec_names,
|
||||
codec_types=codec_types,
|
||||
comments=info_json.get('comments'),
|
||||
description=self.file_group.c_text or self.get_video_description(),
|
||||
id=self.id,
|
||||
info_json_file=self.info_json_file,
|
||||
|
|
@ -304,6 +307,14 @@ class Video(ModelHelper, Base):
|
|||
.filter(FileGroup.primary_path == path).one_or_none()
|
||||
return video
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_url(url: str, session: Session) -> Optional['Video']:
|
||||
video = session.query(Video) \
|
||||
.join(FileGroup, FileGroup.id == Video.file_group_id) \
|
||||
.filter(FileGroup.url == url).one_or_none()
|
||||
return video
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional['Video']:
|
||||
|
|
|
|||
|
|
@ -244,10 +244,9 @@ async def post_download(_: Request, body: schema.DownloadRequest):
|
|||
|
||||
# Don't overwrite settings if restarting download.
|
||||
settings = None
|
||||
if excluded_urls or destination or body.tag_names or body.suffix or body.depth:
|
||||
if excluded_urls or destination or body.tag_names or body.suffix or body.depth or body.do_not_download:
|
||||
settings = dict(excluded_urls=excluded_urls, destination=destination, tag_names=body.tag_names,
|
||||
suffix=body.suffix,
|
||||
depth=body.depth)
|
||||
suffix=body.suffix, do_not_download=body.do_not_download, depth=body.depth)
|
||||
|
||||
if body.frequency:
|
||||
download_manager.recurring_download(body.urls[0], body.frequency, downloader_name=body.downloader,
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ class DownloadRequest:
|
|||
suffix: Optional[str] = None
|
||||
depth: Optional[int] = None
|
||||
max_pages: Optional[int] = None
|
||||
do_not_download: Optional[bool] = False
|
||||
|
||||
def __post_init__(self):
|
||||
urls = [j for i in self.urls if (j := i.strip())]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue