mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Creating background process that fetches comments for Videos that do not yet have comments.
Displaying star for Uploader's comments, heart for favorited comments. Adding wrolpi help files desktop shortcut to rpi image. Using zim file names as FileGroup.title. Removing accidental output commit.
This commit is contained in:
parent
df849ffb42
commit
758ff8e810
14 changed files with 263 additions and 127 deletions
|
|
@ -69,7 +69,7 @@ jobs:
|
|||
POSTGRES_USER: postgres
|
||||
POSTGRES_DB: wrolpi
|
||||
POSTGRES_PASSWORD: "wrolpi"
|
||||
resource_class: small
|
||||
resource_class: medium
|
||||
steps:
|
||||
- checkout
|
||||
- run: sudo apt-get update
|
||||
|
|
@ -96,7 +96,7 @@ jobs:
|
|||
POSTGRES_USER: postgres
|
||||
POSTGRES_DB: wrolpi
|
||||
POSTGRES_PASSWORD: "wrolpi"
|
||||
resource_class: small
|
||||
resource_class: medium
|
||||
steps:
|
||||
- checkout
|
||||
- run: sudo apt-get update
|
||||
|
|
@ -123,7 +123,7 @@ jobs:
|
|||
POSTGRES_USER: postgres
|
||||
POSTGRES_DB: wrolpi
|
||||
POSTGRES_PASSWORD: "wrolpi"
|
||||
resource_class: small
|
||||
resource_class: medium
|
||||
steps:
|
||||
- checkout
|
||||
- run: sudo apt-get update
|
||||
|
|
@ -150,7 +150,7 @@ jobs:
|
|||
POSTGRES_USER: postgres
|
||||
POSTGRES_DB: wrolpi
|
||||
POSTGRES_PASSWORD: "wrolpi"
|
||||
resource_class: small
|
||||
resource_class: medium
|
||||
steps:
|
||||
- checkout
|
||||
- run: sudo apt-get update
|
||||
|
|
@ -177,7 +177,7 @@ jobs:
|
|||
POSTGRES_USER: postgres
|
||||
POSTGRES_DB: wrolpi
|
||||
POSTGRES_PASSWORD: "wrolpi"
|
||||
resource_class: small
|
||||
resource_class: medium
|
||||
steps:
|
||||
- checkout
|
||||
- run: sudo apt-get update
|
||||
|
|
|
|||
35
alembic/versions/d96a04eb2785_video_have_comments.py
Normal file
35
alembic/versions/d96a04eb2785_video_have_comments.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
"""Video.have_comments
|
||||
|
||||
Revision ID: d96a04eb2785
|
||||
Revises: 365c985893b8
|
||||
Create Date: 2024-02-27 16:57:29.831210
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'd96a04eb2785'
|
||||
down_revision = '365c985893b8'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE video ADD COLUMN have_comments BOOLEAN DEFAULT FALSE')
|
||||
session.execute('ALTER TABLE video ADD COLUMN comments_failed BOOLEAN DEFAULT FALSE')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS have_comments')
|
||||
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS comments_failed')
|
||||
|
|
@ -1653,3 +1653,11 @@ export function getParentDirectory(filePath) {
|
|||
|
||||
return parentDirectory;
|
||||
}
|
||||
|
||||
export function MultilineText({text, ...props}) {
|
||||
return <div {...props}>
|
||||
{text.split('\n').map((line, index, array) =>
|
||||
index === array.length - 1 ? line : <p>{line}</p>
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
humanFileSize,
|
||||
humanNumber,
|
||||
isoDatetimeToString,
|
||||
MultilineText,
|
||||
PageContainer,
|
||||
PreviewPath,
|
||||
useTitle
|
||||
|
|
@ -99,20 +100,33 @@ const NoComments = ({video}) => {
|
|||
const VideoComment = ({comment, children}) => {
|
||||
const {t} = React.useContext(ThemeContext);
|
||||
|
||||
const favoriteComment = <Icon name='heart'/>;
|
||||
const likesCount = <React.Fragment>
|
||||
<Icon name='thumbs up'/> {comment['like_count']}
|
||||
</React.Fragment>;
|
||||
const {is_favorited, like_count, author, timestamp, text, author_is_uploader} = comment;
|
||||
|
||||
// Author comments and favorited comments are important.
|
||||
let specialIcon = null;
|
||||
if (author_is_uploader) {
|
||||
specialIcon = <Icon name='star' color='green'/>;
|
||||
} else if (is_favorited) {
|
||||
specialIcon = <Icon name='heart' color='red'/>;
|
||||
}
|
||||
|
||||
const dateElm = <div {...t}>{isoDatetimeToString(timestamp * 1000)}</div>;
|
||||
const likesElm = like_count && <div {...t}>
|
||||
<Icon name='thumbs up'/>{comment['like_count']}
|
||||
</div>;
|
||||
|
||||
return <Comment>
|
||||
<Comment.Content>
|
||||
<Comment.Author as='a'><span {...t}>{comment['author']}</span></Comment.Author>
|
||||
<Comment.Author as='a'>
|
||||
<span {...t}>{specialIcon}{author}</span>
|
||||
</Comment.Author>
|
||||
<Comment.Metadata>
|
||||
<div {...t}>{comment['is_favorited'] && favoriteComment}</div>
|
||||
<div {...t}>{isoDatetimeToString(comment['timestamp'] * 1000)}</div>
|
||||
<div {...t}>{comment['like_count'] && likesCount}</div>
|
||||
{dateElm}
|
||||
{likesElm}
|
||||
</Comment.Metadata>
|
||||
<Comment.Text {...t}>{comment['text']}</Comment.Text>
|
||||
<Comment.Text {...t}>
|
||||
<MultilineText text={text} style={{marginLeft: '0.5em'}}/>
|
||||
</Comment.Text>
|
||||
</Comment.Content>
|
||||
{children && !_.isEmpty(children) &&
|
||||
<CommentGroup>
|
||||
|
|
|
|||
18
main.py
18
main.py
|
|
@ -11,6 +11,7 @@ from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS
|
|||
from wrolpi import root_api, admin
|
||||
from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \
|
||||
wrol_mode_enabled, cancel_refresh_tasks, set_log_level, background_task, cancel_background_tasks
|
||||
from wrolpi.dates import Seconds
|
||||
from wrolpi.downloader import download_manager, import_downloads_config
|
||||
from wrolpi.root_api import api_app
|
||||
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
|
||||
|
|
@ -201,6 +202,23 @@ async def startup(app: Sanic):
|
|||
from modules.zim.lib import flag_outdated_zim_files
|
||||
flag_outdated_zim_files()
|
||||
|
||||
from modules.videos.video.lib import get_missing_videos_comments
|
||||
|
||||
async def periodic_start_video_missing_comments_download():
|
||||
async with flags.refresh_complete.wait_for():
|
||||
# We can't search for Videos missing comments until the refresh has completed.
|
||||
pass
|
||||
|
||||
while True:
|
||||
# Fetch comments for videos every hour.
|
||||
if download_manager.disabled.is_set() or download_manager.stopped.is_set():
|
||||
await asyncio.sleep(10)
|
||||
else:
|
||||
await get_missing_videos_comments()
|
||||
await asyncio.sleep(int(Seconds.hour))
|
||||
|
||||
background_task(periodic_start_video_missing_comments_download())
|
||||
|
||||
|
||||
@api_app.after_server_start
|
||||
async def periodic_downloads(app: Sanic):
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import tempfile
|
||||
import traceback
|
||||
from abc import ABC
|
||||
from typing import Tuple, List, Dict
|
||||
|
|
@ -20,7 +19,7 @@ from wrolpi.db import get_db_session
|
|||
from wrolpi.db import optional_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
|
|
@ -30,6 +29,7 @@ from .lib import get_downloader_config
|
|||
from .models import Video, Channel
|
||||
from .normalize_video_url import normalize_video_url
|
||||
from .schema import ChannelPostRequest
|
||||
from .video.lib import download_video_info_json
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
ydl_logger = logger.getChild('youtube-dl')
|
||||
|
|
@ -469,7 +469,8 @@ class VideoDownloader(Downloader, ABC):
|
|||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
return final_filename, entry
|
||||
|
||||
async def download_info_json(self, download: Download, video_path: pathlib.Path) -> DownloadResult:
|
||||
@staticmethod
|
||||
async def download_info_json(download: Download, video_path: pathlib.Path) -> DownloadResult:
|
||||
"""User has requested to refresh Video metadata. Download only metadata files to temporary directory.
|
||||
Replace existing metadata files only after successful download."""
|
||||
url = download.url
|
||||
|
|
@ -489,104 +490,25 @@ class VideoDownloader(Downloader, ABC):
|
|||
if video.channel_id:
|
||||
location = f'/videos/channel/{video.channel_id}/video/{video.id}'
|
||||
|
||||
# Download metadata files into temporary directory so yt-dlp always downloads the latest files.
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp_dir = pathlib.Path(tmp_dir)
|
||||
# Download video info json directly using yt-dlp.
|
||||
try:
|
||||
info_json = download_video_info_json(url)
|
||||
except Exception as e:
|
||||
logger.error('Failed to download video info json', exc_info=e)
|
||||
return DownloadResult(success=False, error='Failed to download video info json')
|
||||
|
||||
# Download the metadata files, but skip the video file.
|
||||
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
|
||||
cmd = (
|
||||
str(YT_DLP_BIN),
|
||||
'-f', PREFERRED_VIDEO_FORMAT,
|
||||
'--skip-download', # SKIP THE VIDEO FILE.
|
||||
'--match-filter', '!is_live', # Do not attempt to download Live videos.
|
||||
'--write-info-json',
|
||||
'--merge-output-format', PREFERRED_VIDEO_EXTENSION,
|
||||
'--remux-video', PREFERRED_VIDEO_EXTENSION,
|
||||
'-o', file_name_format,
|
||||
'--no-cache-dir',
|
||||
'--compat-options', 'no-live-chat',
|
||||
# Get top 20 comments, 10 replies per parent.
|
||||
'--write-comments',
|
||||
'--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
|
||||
url,
|
||||
)
|
||||
return_code, logs, _ = await self.process_runner(url, cmd, tmp_dir)
|
||||
with get_db_session(commit=True) as session:
|
||||
# Find the existing video, replace its info json.
|
||||
video = Video.get_by_url(url, session)
|
||||
video.replace_info_json(info_json)
|
||||
|
||||
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
|
||||
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
|
||||
if video.get_comments():
|
||||
video.have_comments = True
|
||||
|
||||
if return_code != 0:
|
||||
error = f'{stdout}\n\n\n{stderr}\n\nvideo info json downloader process exited with {return_code}'
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error=error,
|
||||
location=location,
|
||||
)
|
||||
logger.debug(f'Downloaded video info json {location=}')
|
||||
logger.info(f'Successfully downloaded video info json {url} {video}')
|
||||
|
||||
# Find the info json file in the temporary directory.
|
||||
info_json_paths = [i for i in tmp_dir.iterdir() if i.name.endswith('.info.json') and i.stat().st_size]
|
||||
if len(info_json_paths) == 0:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='No info json file was downloaded.',
|
||||
location=location,
|
||||
)
|
||||
elif len(info_json_paths) > 1:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='More than one info json file was downloaded.',
|
||||
location=location,
|
||||
)
|
||||
|
||||
info_json_path = info_json_paths[0]
|
||||
|
||||
if not info_json_path.stat().st_size:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Video info json file was empty',
|
||||
location=location,
|
||||
)
|
||||
|
||||
# Reformat new info json file for human reading.
|
||||
format_json_file(info_json_path)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Find the existing video, replace its info json.
|
||||
video = Video.get_by_url(url, session)
|
||||
|
||||
if video.info_json_path and video.info_json_path.is_file():
|
||||
# Move old file temporarily. Only delete old data if the swap is successful.
|
||||
logger.debug(f'Replacing info json file: {video.info_json_path}')
|
||||
tmp_info_json_path = pathlib.Path(f'{video.info_json_path}.tmp')
|
||||
if tmp_info_json_path.is_file():
|
||||
if video.info_json_path.stat().st_size:
|
||||
tmp_info_json_path.unlink()
|
||||
else:
|
||||
raise UnrecoverableDownloadError(
|
||||
f'Temporary info json file already exists: {tmp_info_json_path}')
|
||||
|
||||
# Move existing info json file to a temporary location. Move it back if this fails.
|
||||
video.info_json_path.rename(tmp_info_json_path)
|
||||
try:
|
||||
video.info_json_path.write_text(info_json_path.read_text())
|
||||
tmp_info_json_path.unlink()
|
||||
except Exception as e:
|
||||
logger.error('Failed to swap video info json file', exc_info=e)
|
||||
tmp_info_json_path.rename(video.info_json_path)
|
||||
raise
|
||||
else:
|
||||
# info json file does not exist, write one next to the video.
|
||||
video_path = video.video_path
|
||||
stem, _ = split_path_stem_and_suffix(video_path)
|
||||
new_info_json_path = pathlib.Path(f'{stem}.info.json')
|
||||
new_info_json_path.write_text(info_json_path.read_text())
|
||||
video.file_group.append_files(new_info_json_path)
|
||||
|
||||
logger.debug(f'Downloaded video info json {location=}')
|
||||
logger.info(f'Successfully downloaded video info json {url} {video}')
|
||||
|
||||
return DownloadResult(success=True, location=location)
|
||||
return DownloadResult(success=True, location=location)
|
||||
|
||||
|
||||
channel_downloader = ChannelDownloader()
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ class Video(ModelHelper, Base):
|
|||
source_id = Column(String) # The id from yt-dlp
|
||||
view_count = Column(Integer) # The view count from the ChannelDownloader (or from initial download)
|
||||
ffprobe_json = deferred(Column(JSON)) # Data that is fetched once from ffprobe (ffmpeg)
|
||||
have_comments = Column(Boolean, default=False) # see `get_missing_videos_comments`
|
||||
comments_failed = Column(Boolean, default=False) # see `get_missing_videos_comments`
|
||||
|
||||
channel_id = Column(Integer, ForeignKey('channel.id'))
|
||||
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
|
||||
|
|
@ -60,8 +62,6 @@ class Video(ModelHelper, Base):
|
|||
except Exception as e:
|
||||
logger.error(f'{self} ffprobe_json is invalid', exc_info=e)
|
||||
|
||||
info_json = self.get_info_json() or dict()
|
||||
|
||||
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
|
||||
d['video'] = dict(
|
||||
caption=self.file_group.d_text,
|
||||
|
|
@ -70,7 +70,7 @@ class Video(ModelHelper, Base):
|
|||
channel_id=self.channel_id,
|
||||
codec_names=codec_names,
|
||||
codec_types=codec_types,
|
||||
comments=info_json.get('comments'),
|
||||
comments=self.get_comments(),
|
||||
description=self.file_group.c_text or self.get_video_description(),
|
||||
id=self.id,
|
||||
info_json_file=self.info_json_file,
|
||||
|
|
@ -102,7 +102,7 @@ class Video(ModelHelper, Base):
|
|||
logger.warning(f'{self} cannot be added to skip list because it does not have a URL')
|
||||
|
||||
def get_info_json(self) -> Optional[Dict]:
|
||||
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
|
||||
"""If this Video has an info_json file, return its contents. Otherwise, return None."""
|
||||
info_json_path = self.info_json_path
|
||||
if not info_json_path:
|
||||
return
|
||||
|
|
@ -121,12 +121,8 @@ class Video(ModelHelper, Base):
|
|||
"""
|
||||
Get the Video description from the file system.
|
||||
"""
|
||||
# First try to get description from info_json file.
|
||||
info_json = self.get_info_json()
|
||||
if info_json:
|
||||
description = info_json.get('description')
|
||||
if description:
|
||||
return description
|
||||
if (info_json := self.get_info_json()) and (description := info_json.get('description')):
|
||||
return description
|
||||
|
||||
def get_surrounding_videos(self):
|
||||
"""
|
||||
|
|
@ -231,6 +227,9 @@ class Video(ModelHelper, Base):
|
|||
self.file_group.c_text = self.get_video_description()
|
||||
# self.file_group.d_text is handled in `validate_video`.
|
||||
|
||||
if (comments := self.get_comments()) and len(comments) >= 5:
|
||||
self.have_comments = True
|
||||
|
||||
@staticmethod
|
||||
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
|
|
@ -477,6 +476,17 @@ class Video(ModelHelper, Base):
|
|||
return f'/videos/channel/{self.channel_id}/video/{self.id}'
|
||||
return f'/videos/video/{self.id}'
|
||||
|
||||
def get_comments(self):
|
||||
return (self.get_info_json() or dict()).get('comments')
|
||||
|
||||
def replace_info_json(self, info_json: dict):
|
||||
"""Replace the info json file with the new json dict. Handles adding new info_json file, if necessary."""
|
||||
info_json_path = self.info_json_path or self.video_path.with_suffix('.info.json')
|
||||
info_json_path.write_text(json.dumps(info_json, indent=2))
|
||||
|
||||
if not self.info_json_path:
|
||||
self.file_group.append_files(info_json_path)
|
||||
|
||||
|
||||
class Channel(ModelHelper, Base):
|
||||
__tablename__ = 'channel'
|
||||
|
|
|
|||
|
|
@ -1,12 +1,22 @@
|
|||
import asyncio
|
||||
import json
|
||||
import random
|
||||
from datetime import timedelta
|
||||
from typing import Tuple, Optional, List, Dict
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.common import run_after, logger
|
||||
from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, optional_session
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.files.lib import handle_file_group_search_results
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import tag_append_sub_select_where
|
||||
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT
|
||||
from ..errors import UnknownVideo
|
||||
from ..lib import save_channels_config
|
||||
|
||||
|
|
@ -157,3 +167,100 @@ def delete_videos(*video_ids: int, session: Session = None):
|
|||
for video in videos:
|
||||
video.delete()
|
||||
session.commit()
|
||||
|
||||
|
||||
def download_video_info_json(url: str) -> dict:
|
||||
ydl_opts = dict(
|
||||
getcomments=True,
|
||||
skip_download=True,
|
||||
extractor_args={'youtube': {'max_comments': ['all', '20', 'all', '10'], 'comment_sort': ['top']}},
|
||||
)
|
||||
|
||||
ydl_logger = logger.getChild('youtube-dl')
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.params['logger'] = ydl_logger
|
||||
info = ydl.extract_info(url, download=False)
|
||||
ydl.sanitize_info(info)
|
||||
return info
|
||||
|
||||
|
||||
@limit_concurrent(1)
|
||||
@wrol_mode_check
|
||||
async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
|
||||
"""
|
||||
Fetches Video info json for videos without comments, if comments are found then the info json file is replaced.
|
||||
"""
|
||||
one_month_ago = now() - timedelta(days=30)
|
||||
|
||||
# Get any videos over a month old that do not have comments.
|
||||
with get_db_session() as session:
|
||||
videos = session.query(Video).filter(
|
||||
# Have yet to get comments.
|
||||
Video.have_comments == False,
|
||||
Video.comments_failed == False,
|
||||
# Need a URL to try to get comments.
|
||||
FileGroup.url != None, # noqa
|
||||
# We want old videos (time for comments to accumulate), or those which we don't know the published date.
|
||||
or_(
|
||||
one_month_ago > FileGroup.published_datetime,
|
||||
FileGroup.published_datetime == None,
|
||||
),
|
||||
).join(FileGroup) \
|
||||
.order_by(FileGroup.published_datetime.nullsfirst()) \
|
||||
.limit(limit)
|
||||
video_urls = [i.file_group.url for i in videos]
|
||||
|
||||
logger.info(f'Found {len(video_urls)} videos missing comments')
|
||||
|
||||
def add_video_to_skip_list(video_url: str):
|
||||
"""Do not attempt to get comments of the provided Video again."""
|
||||
with get_db_session(commit=True) as session_:
|
||||
for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all():
|
||||
video_.comments_failed = True
|
||||
|
||||
for url in video_urls:
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
if download_manager.is_skipped(url):
|
||||
# This video is skipped, do not try to get comments again.
|
||||
add_video_to_skip_list(url)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
have_comments = False
|
||||
for video in session.query(Video).join(FileGroup).filter_by(url=url).all():
|
||||
if (comments := video.get_comments()) and len(comments) >= 5:
|
||||
# We already have enough comments. DB may have been wiped, we don't want to fetch comments again.
|
||||
logger.info(f'Already have comments for Video: {url=}')
|
||||
have_comments = video.have_comments = True
|
||||
|
||||
if have_comments:
|
||||
download_manager.add_to_skip_list(url)
|
||||
continue
|
||||
|
||||
tries = 5
|
||||
while tries > 0:
|
||||
try:
|
||||
tries -= 1
|
||||
# Get info json about the video. Retry until we have a list of comments.
|
||||
info = download_video_info_json(url)
|
||||
if info and isinstance(info.get('comments'), list):
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f'Got error when attempting to download comments: {url=}', exc_info=e)
|
||||
else:
|
||||
logger.error(f'Never got comments for video: {url=}')
|
||||
download_manager.add_to_skip_list(url)
|
||||
add_video_to_skip_list(url)
|
||||
continue
|
||||
|
||||
logger.debug(f'Got {len(info["comments"])} comments for Video: {url=}')
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for video in session.query(Video).join(FileGroup).filter_by(url=url).all():
|
||||
video.replace_info_json(info)
|
||||
video.have_comments = True
|
||||
|
||||
# Sleep a random amount of time, so we don't spam.
|
||||
await asyncio.sleep(random.randint(2, 20))
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ def zim_path_factory(test_zim_bytes, test_directory):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def test_zim(test_session, zim_path_factory, test_directory) -> Zim:
|
||||
def test_zim(test_session, zim_path_factory) -> Zim:
|
||||
zim = Zim.from_paths(test_session, zim_path_factory())
|
||||
return zim
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from wrolpi.common import register_modeler, logger, extract_html_text, extract_h
|
|||
register_refresh_cleanup, background_task
|
||||
from wrolpi.db import get_db_session, optional_session, get_db_curs
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.lib import refresh_files, split_file_name_words
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED
|
||||
|
||||
|
|
@ -62,6 +62,8 @@ async def zim_modeler():
|
|||
session.add(zim)
|
||||
session.flush([zim])
|
||||
zim_id = zim.id
|
||||
file_group.title = file_group.primary_path.name
|
||||
file_group.a_text = split_file_name_words(file_group.primary_path.name)
|
||||
file_group.indexed = True
|
||||
except Exception as e:
|
||||
if PYTEST:
|
||||
|
|
|
|||
|
|
@ -328,3 +328,16 @@ async def test_zim_subscription_download_import(test_session, test_downloader_co
|
|||
assert subscription.download_id == recurring.id
|
||||
assert subscription.name == 'Wikisource'
|
||||
assert subscription.language == 'en'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_modeler(test_session, zim_factory):
|
||||
"""Zim modeler sets FileGroup title and a_text."""
|
||||
zim_factory('the_zim_title')
|
||||
test_session.commit()
|
||||
|
||||
await files_lib.refresh_files()
|
||||
|
||||
fg: FileGroup = test_session.query(FileGroup).one()
|
||||
assert fg.title == 'the_zim_title'
|
||||
assert fg.a_text == 'the zim title'
|
||||
|
|
|
|||
|
|
@ -32,6 +32,14 @@ Type=Link
|
|||
URL=http://127.0.0.1:8086
|
||||
Icon=/opt/wrolpi-help/venv/lib/python3.11/site-packages/mkdocs/themes/mkdocs/img/favicon.ico
|
||||
EOF
|
||||
cat >/home/pi/Desktop/wrolpi-help-files.desktop <<'EOF'
|
||||
[Desktop Entry]
|
||||
Encoding=UTF-8
|
||||
Name=WROLPi Help Files
|
||||
Type=Link
|
||||
URL=/opt/wrolpi-help/docs/
|
||||
Icon=/usr/share/icons/gnome/48x48/places/folder.png
|
||||
EOF
|
||||
chown -R pi:pi /home/pi
|
||||
chmod 0600 /home/pi/.pgpass
|
||||
usermod -aG wrolpi pi
|
||||
|
|
|
|||
|
|
@ -80,9 +80,6 @@ def get_db_context() -> Tuple[sqlalchemy.create_engine, Session]:
|
|||
return local_engine, session
|
||||
|
||||
|
||||
# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_db_session(commit: bool = False) -> ContextManager[Session]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100))
|
|||
FILE_MAX_PDF_SIZE = int(os.environ.get('FILE_MAX_PDF_SIZE', 40_000_000))
|
||||
FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000))
|
||||
|
||||
VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 50))
|
||||
|
||||
DEFAULT_HTTP_HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36',
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue