From 758ff8e810b30a47d5ad0498e40b6c416e92560f Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 27 Feb 2024 18:54:14 -0700 Subject: [PATCH] Creating background process that fetches comments for Videos that do not yet have comments. Displaying star for Uploader's comments, heart for favorited comments. Adding wrolpi help files desktop shortcut to rpi image. Using zim file names as FileGroup.title. Removing accidental output commit. --- .circleci/config.yml | 10 +- .../d96a04eb2785_video_have_comments.py | 35 ++++++ app/src/components/Common.js | 8 ++ app/src/components/VideoPlayer.js | 32 +++-- main.py | 18 +++ modules/videos/downloader.py | 116 +++--------------- modules/videos/models.py | 30 +++-- modules/videos/video/lib.py | 109 +++++++++++++++- modules/zim/conftest.py | 2 +- modules/zim/lib.py | 4 +- modules/zim/test/test_lib.py | 13 ++ pi-gen/stage2/04-wrolpi/04-run-chroot.sh | 8 ++ wrolpi/db.py | 3 - wrolpi/vars.py | 2 + 14 files changed, 263 insertions(+), 127 deletions(-) create mode 100644 alembic/versions/d96a04eb2785_video_have_comments.py diff --git a/.circleci/config.yml b/.circleci/config.yml index e7946204..acb7e620 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -96,7 +96,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -123,7 +123,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -150,7 +150,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update @@ -177,7 +177,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: small + resource_class: medium steps: - checkout - run: sudo apt-get update diff --git a/alembic/versions/d96a04eb2785_video_have_comments.py b/alembic/versions/d96a04eb2785_video_have_comments.py new file mode 100644 index 00000000..f29aa1af --- /dev/null +++ b/alembic/versions/d96a04eb2785_video_have_comments.py @@ -0,0 +1,35 @@ +"""Video.have_comments + +Revision ID: d96a04eb2785 +Revises: 365c985893b8 +Create Date: 2024-02-27 16:57:29.831210 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'd96a04eb2785' +down_revision = '365c985893b8' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video ADD COLUMN have_comments BOOLEAN DEFAULT FALSE') + session.execute('ALTER TABLE video ADD COLUMN comments_failed BOOLEAN DEFAULT FALSE') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS have_comments') + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS comments_failed') diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 0bea8ef8..f9ac7288 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -1653,3 +1653,11 @@ export function getParentDirectory(filePath) { return parentDirectory; } + +export function MultilineText({text, ...props}) { + return
+ {text.split('\n').map((line, index, array) => + index === array.length - 1 ? line :

{line}

+ )} +
+} diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 555fbcb4..ecb6473f 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -10,6 +10,7 @@ import { humanFileSize, humanNumber, isoDatetimeToString, + MultilineText, PageContainer, PreviewPath, useTitle @@ -99,20 +100,33 @@ const NoComments = ({video}) => { const VideoComment = ({comment, children}) => { const {t} = React.useContext(ThemeContext); - const favoriteComment = ; - const likesCount = - {comment['like_count']} - ; + const {is_favorited, like_count, author, timestamp, text, author_is_uploader} = comment; + + // Author comments and favorited comments are important. + let specialIcon = null; + if (author_is_uploader) { + specialIcon = ; + } else if (is_favorited) { + specialIcon = ; + } + + const dateElm =
{isoDatetimeToString(timestamp * 1000)}
; + const likesElm = like_count &&
+ {comment['like_count']} +
; return - {comment['author']} + + {specialIcon}{author} + -
{comment['is_favorited'] && favoriteComment}
-
{isoDatetimeToString(comment['timestamp'] * 1000)}
-
{comment['like_count'] && likesCount}
+ {dateElm} + {likesElm}
- {comment['text']} + + +
{children && !_.isEmpty(children) && diff --git a/main.py b/main.py index 9bd19c3d..3f3d2df6 100755 --- a/main.py +++ b/main.py @@ -11,6 +11,7 @@ from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS from wrolpi import root_api, admin from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \ wrol_mode_enabled, cancel_refresh_tasks, set_log_level, background_task, cancel_background_tasks +from wrolpi.dates import Seconds from wrolpi.downloader import download_manager, import_downloads_config from wrolpi.root_api import api_app from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST @@ -201,6 +202,23 @@ async def startup(app: Sanic): from modules.zim.lib import flag_outdated_zim_files flag_outdated_zim_files() + from modules.videos.video.lib import get_missing_videos_comments + + async def periodic_start_video_missing_comments_download(): + async with flags.refresh_complete.wait_for(): + # We can't search for Videos missing comments until the refresh has completed. + pass + + while True: + # Fetch comments for videos every hour. + if download_manager.disabled.is_set() or download_manager.stopped.is_set(): + await asyncio.sleep(10) + else: + await get_missing_videos_comments() + await asyncio.sleep(int(Seconds.hour)) + + background_task(periodic_start_video_missing_comments_download()) + @api_app.after_server_start async def periodic_downloads(app: Sanic): diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 2785fb0d..6286e050 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -2,7 +2,6 @@ import json import pathlib import re -import tempfile import traceback from abc import ABC from typing import Tuple, List, Dict @@ -20,7 +19,7 @@ from wrolpi.db import get_db_session from wrolpi.db import optional_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError -from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix +from wrolpi.files.lib import glob_shared_stem from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel @@ -30,6 +29,7 @@ from .lib import get_downloader_config from .models import Video, Channel from .normalize_video_url import normalize_video_url from .schema import ChannelPostRequest +from .video.lib import download_video_info_json logger = logger.getChild(__name__) ydl_logger = logger.getChild('youtube-dl') @@ -469,7 +469,8 @@ class VideoDownloader(Downloader, ABC): logger.debug(f'Downloading {url} to {out_dir}') return final_filename, entry - async def download_info_json(self, download: Download, video_path: pathlib.Path) -> DownloadResult: + @staticmethod + async def download_info_json(download: Download, video_path: pathlib.Path) -> DownloadResult: """User has requested to refresh Video metadata. Download only metadata files to temporary directory. Replace existing metadata files only after successful download.""" url = download.url @@ -489,104 +490,25 @@ class VideoDownloader(Downloader, ABC): if video.channel_id: location = f'/videos/channel/{video.channel_id}/video/{video.id}' - # Download metadata files into temporary directory so yt-dlp always downloads the latest files. - with tempfile.TemporaryDirectory() as tmp_dir: - tmp_dir = pathlib.Path(tmp_dir) + # Download video info json directly using yt-dlp. + try: + info_json = download_video_info_json(url) + except Exception as e: + logger.error('Failed to download video info json', exc_info=e) + return DownloadResult(success=False, error='Failed to download video info json') - # Download the metadata files, but skip the video file. - file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' - cmd = ( - str(YT_DLP_BIN), - '-f', PREFERRED_VIDEO_FORMAT, - '--skip-download', # SKIP THE VIDEO FILE. - '--match-filter', '!is_live', # Do not attempt to download Live videos. - '--write-info-json', - '--merge-output-format', PREFERRED_VIDEO_EXTENSION, - '--remux-video', PREFERRED_VIDEO_EXTENSION, - '-o', file_name_format, - '--no-cache-dir', - '--compat-options', 'no-live-chat', - # Get top 20 comments, 10 replies per parent. - '--write-comments', - '--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top', - url, - ) - return_code, logs, _ = await self.process_runner(url, cmd, tmp_dir) + with get_db_session(commit=True) as session: + # Find the existing video, replace its info json. + video = Video.get_by_url(url, session) + video.replace_info_json(info_json) - stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr'] - stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout'] + if video.get_comments(): + video.have_comments = True - if return_code != 0: - error = f'{stdout}\n\n\n{stderr}\n\nvideo info json downloader process exited with {return_code}' - return DownloadResult( - success=False, - error=error, - location=location, - ) + logger.debug(f'Downloaded video info json {location=}') + logger.info(f'Successfully downloaded video info json {url} {video}') - # Find the info json file in the temporary directory. - info_json_paths = [i for i in tmp_dir.iterdir() if i.name.endswith('.info.json') and i.stat().st_size] - if len(info_json_paths) == 0: - return DownloadResult( - success=False, - error='No info json file was downloaded.', - location=location, - ) - elif len(info_json_paths) > 1: - return DownloadResult( - success=False, - error='More than one info json file was downloaded.', - location=location, - ) - - info_json_path = info_json_paths[0] - - if not info_json_path.stat().st_size: - return DownloadResult( - success=False, - error='Video info json file was empty', - location=location, - ) - - # Reformat new info json file for human reading. - format_json_file(info_json_path) - - with get_db_session(commit=True) as session: - # Find the existing video, replace its info json. - video = Video.get_by_url(url, session) - - if video.info_json_path and video.info_json_path.is_file(): - # Move old file temporarily. Only delete old data if the swap is successful. - logger.debug(f'Replacing info json file: {video.info_json_path}') - tmp_info_json_path = pathlib.Path(f'{video.info_json_path}.tmp') - if tmp_info_json_path.is_file(): - if video.info_json_path.stat().st_size: - tmp_info_json_path.unlink() - else: - raise UnrecoverableDownloadError( - f'Temporary info json file already exists: {tmp_info_json_path}') - - # Move existing info json file to a temporary location. Move it back if this fails. - video.info_json_path.rename(tmp_info_json_path) - try: - video.info_json_path.write_text(info_json_path.read_text()) - tmp_info_json_path.unlink() - except Exception as e: - logger.error('Failed to swap video info json file', exc_info=e) - tmp_info_json_path.rename(video.info_json_path) - raise - else: - # info json file does not exist, write one next to the video. - video_path = video.video_path - stem, _ = split_path_stem_and_suffix(video_path) - new_info_json_path = pathlib.Path(f'{stem}.info.json') - new_info_json_path.write_text(info_json_path.read_text()) - video.file_group.append_files(new_info_json_path) - - logger.debug(f'Downloaded video info json {location=}') - logger.info(f'Successfully downloaded video info json {url} {video}') - - return DownloadResult(success=True, location=location) + return DownloadResult(success=True, location=location) channel_downloader = ChannelDownloader() diff --git a/modules/videos/models.py b/modules/videos/models.py index dc38f219..f27f81ca 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -30,6 +30,8 @@ class Video(ModelHelper, Base): source_id = Column(String) # The id from yt-dlp view_count = Column(Integer) # The view count from the ChannelDownloader (or from initial download) ffprobe_json = deferred(Column(JSON)) # Data that is fetched once from ffprobe (ffmpeg) + have_comments = Column(Boolean, default=False) # see `get_missing_videos_comments` + comments_failed = Column(Boolean, default=False) # see `get_missing_videos_comments` channel_id = Column(Integer, ForeignKey('channel.id')) channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos') @@ -60,8 +62,6 @@ class Video(ModelHelper, Base): except Exception as e: logger.error(f'{self} ffprobe_json is invalid', exc_info=e) - info_json = self.get_info_json() or dict() - # Put live data in "video" instead of "data" to avoid confusion on the frontend. d['video'] = dict( caption=self.file_group.d_text, @@ -70,7 +70,7 @@ class Video(ModelHelper, Base): channel_id=self.channel_id, codec_names=codec_names, codec_types=codec_types, - comments=info_json.get('comments'), + comments=self.get_comments(), description=self.file_group.c_text or self.get_video_description(), id=self.id, info_json_file=self.info_json_file, @@ -102,7 +102,7 @@ class Video(ModelHelper, Base): logger.warning(f'{self} cannot be added to skip list because it does not have a URL') def get_info_json(self) -> Optional[Dict]: - """If this Video has an info_json file, return it's contents. Otherwise, return None.""" + """If this Video has an info_json file, return its contents. Otherwise, return None.""" info_json_path = self.info_json_path if not info_json_path: return @@ -121,12 +121,8 @@ class Video(ModelHelper, Base): """ Get the Video description from the file system. """ - # First try to get description from info_json file. - info_json = self.get_info_json() - if info_json: - description = info_json.get('description') - if description: - return description + if (info_json := self.get_info_json()) and (description := info_json.get('description')): + return description def get_surrounding_videos(self): """ @@ -231,6 +227,9 @@ class Video(ModelHelper, Base): self.file_group.c_text = self.get_video_description() # self.file_group.d_text is handled in `validate_video`. + if (comments := self.get_comments()) and len(comments) >= 5: + self.have_comments = True + @staticmethod def from_paths(session: Session, *paths: pathlib.Path) -> 'Video': file_group = FileGroup.from_paths(session, *paths) @@ -477,6 +476,17 @@ class Video(ModelHelper, Base): return f'/videos/channel/{self.channel_id}/video/{self.id}' return f'/videos/video/{self.id}' + def get_comments(self): + return (self.get_info_json() or dict()).get('comments') + + def replace_info_json(self, info_json: dict): + """Replace the info json file with the new json dict. Handles adding new info_json file, if necessary.""" + info_json_path = self.info_json_path or self.video_path.with_suffix('.info.json') + info_json_path.write_text(json.dumps(info_json, indent=2)) + + if not self.info_json_path: + self.file_group.append_files(info_json_path) + class Channel(ModelHelper, Base): __tablename__ = 'channel' diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index bef46ed5..63be8cbb 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -1,12 +1,22 @@ +import asyncio +import json +import random +from datetime import timedelta from typing import Tuple, Optional, List, Dict +import yt_dlp +from sqlalchemy import or_ from sqlalchemy.orm import Session from modules.videos.models import Video -from wrolpi.common import run_after, logger +from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check +from wrolpi.dates import now from wrolpi.db import get_db_session, optional_session +from wrolpi.downloader import download_manager from wrolpi.files.lib import handle_file_group_search_results +from wrolpi.files.models import FileGroup from wrolpi.tags import tag_append_sub_select_where +from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT from ..errors import UnknownVideo from ..lib import save_channels_config @@ -157,3 +167,100 @@ def delete_videos(*video_ids: int, session: Session = None): for video in videos: video.delete() session.commit() + + +def download_video_info_json(url: str) -> dict: + ydl_opts = dict( + getcomments=True, + skip_download=True, + extractor_args={'youtube': {'max_comments': ['all', '20', 'all', '10'], 'comment_sort': ['top']}}, + ) + + ydl_logger = logger.getChild('youtube-dl') + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + ydl.params['logger'] = ydl_logger + info = ydl.extract_info(url, download=False) + ydl.sanitize_info(info) + return info + + +@limit_concurrent(1) +@wrol_mode_check +async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT): + """ + Fetches Video info json for videos without comments, if comments are found then the info json file is replaced. + """ + one_month_ago = now() - timedelta(days=30) + + # Get any videos over a month old that do not have comments. + with get_db_session() as session: + videos = session.query(Video).filter( + # Have yet to get comments. + Video.have_comments == False, + Video.comments_failed == False, + # Need a URL to try to get comments. + FileGroup.url != None, # noqa + # We want old videos (time for comments to accumulate), or those which we don't know the published date. + or_( + one_month_ago > FileGroup.published_datetime, + FileGroup.published_datetime == None, + ), + ).join(FileGroup) \ + .order_by(FileGroup.published_datetime.nullsfirst()) \ + .limit(limit) + video_urls = [i.file_group.url for i in videos] + + logger.info(f'Found {len(video_urls)} videos missing comments') + + def add_video_to_skip_list(video_url: str): + """Do not attempt to get comments of the provided Video again.""" + with get_db_session(commit=True) as session_: + for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all(): + video_.comments_failed = True + + for url in video_urls: + # Sleep to catch cancel. + await asyncio.sleep(0) + + if download_manager.is_skipped(url): + # This video is skipped, do not try to get comments again. + add_video_to_skip_list(url) + + with get_db_session(commit=True) as session: + have_comments = False + for video in session.query(Video).join(FileGroup).filter_by(url=url).all(): + if (comments := video.get_comments()) and len(comments) >= 5: + # We already have enough comments. DB may have been wiped, we don't want to fetch comments again. + logger.info(f'Already have comments for Video: {url=}') + have_comments = video.have_comments = True + + if have_comments: + download_manager.add_to_skip_list(url) + continue + + tries = 5 + while tries > 0: + try: + tries -= 1 + # Get info json about the video. Retry until we have a list of comments. + info = download_video_info_json(url) + if info and isinstance(info.get('comments'), list): + break + except Exception as e: + logger.error(f'Got error when attempting to download comments: {url=}', exc_info=e) + else: + logger.error(f'Never got comments for video: {url=}') + download_manager.add_to_skip_list(url) + add_video_to_skip_list(url) + continue + + logger.debug(f'Got {len(info["comments"])} comments for Video: {url=}') + + with get_db_session(commit=True) as session: + for video in session.query(Video).join(FileGroup).filter_by(url=url).all(): + video.replace_info_json(info) + video.have_comments = True + + # Sleep a random amount of time, so we don't spam. + await asyncio.sleep(random.randint(2, 20)) diff --git a/modules/zim/conftest.py b/modules/zim/conftest.py index 493ad18b..5309cf78 100644 --- a/modules/zim/conftest.py +++ b/modules/zim/conftest.py @@ -85,7 +85,7 @@ def zim_path_factory(test_zim_bytes, test_directory): @pytest.fixture -def test_zim(test_session, zim_path_factory, test_directory) -> Zim: +def test_zim(test_session, zim_path_factory) -> Zim: zim = Zim.from_paths(test_session, zim_path_factory()) return zim diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 5fa2c441..55a102a5 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -19,7 +19,7 @@ from wrolpi.common import register_modeler, logger, extract_html_text, extract_h register_refresh_cleanup, background_task from wrolpi.db import get_db_session, optional_session, get_db_curs from wrolpi.downloader import DownloadFrequency -from wrolpi.files.lib import refresh_files +from wrolpi.files.lib import refresh_files, split_file_name_words from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST, DOCKERIZED @@ -62,6 +62,8 @@ async def zim_modeler(): session.add(zim) session.flush([zim]) zim_id = zim.id + file_group.title = file_group.primary_path.name + file_group.a_text = split_file_name_words(file_group.primary_path.name) file_group.indexed = True except Exception as e: if PYTEST: diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index cb7965e2..731fb66b 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -328,3 +328,16 @@ async def test_zim_subscription_download_import(test_session, test_downloader_co assert subscription.download_id == recurring.id assert subscription.name == 'Wikisource' assert subscription.language == 'en' + + +@pytest.mark.asyncio +async def test_zim_modeler(test_session, zim_factory): + """Zim modeler sets FileGroup title and a_text.""" + zim_factory('the_zim_title') + test_session.commit() + + await files_lib.refresh_files() + + fg: FileGroup = test_session.query(FileGroup).one() + assert fg.title == 'the_zim_title' + assert fg.a_text == 'the zim title' diff --git a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh index 044bdb6e..d44b4876 100755 --- a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh @@ -32,6 +32,14 @@ Type=Link URL=http://127.0.0.1:8086 Icon=/opt/wrolpi-help/venv/lib/python3.11/site-packages/mkdocs/themes/mkdocs/img/favicon.ico EOF +cat >/home/pi/Desktop/wrolpi-help-files.desktop <<'EOF' +[Desktop Entry] +Encoding=UTF-8 +Name=WROLPi Help Files +Type=Link +URL=/opt/wrolpi-help/docs/ +Icon=/usr/share/icons/gnome/48x48/places/folder.png +EOF chown -R pi:pi /home/pi chmod 0600 /home/pi/.pgpass usermod -aG wrolpi pi diff --git a/wrolpi/db.py b/wrolpi/db.py index 0b2bf015..12379e02 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -80,9 +80,6 @@ def get_db_context() -> Tuple[sqlalchemy.create_engine, Session]: return local_engine, session -# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) - - @contextmanager def get_db_session(commit: bool = False) -> ContextManager[Session]: """ diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 98a9cadd..c5789d3e 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -75,6 +75,8 @@ FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100)) FILE_MAX_PDF_SIZE = int(os.environ.get('FILE_MAX_PDF_SIZE', 40_000_000)) FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000)) +VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 50)) + DEFAULT_HTTP_HEADERS = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36', }