From e8e5b5727f9a6f1d0b8835ae49e5bd04472a5960 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 8 Aug 2023 13:42:55 -0600 Subject: [PATCH] Creating Video.validated which is used by find_corrupt_videos.py to search. --- alembic/versions/f360a276cdcf_video_valid.py | 31 ++++ app/src/components/FilePreview.js | 37 ++-- app/src/components/VideoPlayer.js | 5 - app/src/components/admin/Downloads.js | 28 +-- app/src/hooks/customHooks.js | 4 +- modules/videos/__init__.py | 3 +- modules/videos/common.py | 48 ++++-- modules/videos/downloader.py | 11 +- modules/videos/lib.py | 7 +- modules/videos/models.py | 10 +- modules/videos/test/test_common.py | 3 +- modules/videos/test/test_downloader.py | 2 +- pi-gen/stage2/04-wrolpi/01-packages | 2 + scripts/find_corrupt_videos.py | 170 +++++++++++++------ scripts/install_debian_11.sh | 2 + scripts/install_raspberrypios.sh | 2 + wrolpi/downloader.py | 13 ++ 17 files changed, 253 insertions(+), 125 deletions(-) create mode 100644 alembic/versions/f360a276cdcf_video_valid.py diff --git a/alembic/versions/f360a276cdcf_video_valid.py b/alembic/versions/f360a276cdcf_video_valid.py new file mode 100644 index 00000000..dea75135 --- /dev/null +++ b/alembic/versions/f360a276cdcf_video_valid.py @@ -0,0 +1,31 @@ +"""Video.valid. + +Revision ID: f360a276cdcf +Revises: 8bdb48a486b9 +Create Date: 2023-08-07 22:49:37.001794 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'f360a276cdcf' +down_revision = '8bdb48a486b9' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS validated') diff --git a/app/src/components/FilePreview.js b/app/src/components/FilePreview.js index 25c0dc8c..c1b0c188 100644 --- a/app/src/components/FilePreview.js +++ b/app/src/components/FilePreview.js @@ -33,7 +33,7 @@ function getEpubViewerURL(previewFile) { } } -function getIframeModal(previewFile) { +function getIframePreviewModal(previewFile) { const url = getMediaPathURL(previewFile); return
@@ -47,7 +47,7 @@ function getIframeModal(previewFile) { } -function getImageModal(previewFile) { +function getImagePreviewModal(previewFile) { const path = previewFile.primary_path ?? previewFile.path; const url = getMediaPathURL(previewFile); // Get the file name from the absolute path. @@ -64,7 +64,7 @@ function getImageModal(previewFile) { } -function getEpubModal(previewFile) { +function getEpubPreviewModal(previewFile) { const downloadURL = getDownloadPathURL(previewFile); const viewerUrl = `/epub/epub.html?url=${downloadURL}`; return @@ -77,11 +77,10 @@ function getEpubModal(previewFile) { } -function getVideoModal(previewFile) { +function getVideoPreviewModal(previewFile) { const url = getMediaPathURL(previewFile); const path = previewFile.primary_path ?? previewFile.path; const name = path.replace(/^.*[\\\/]/, ''); - const type = previewFile.mimetype ? previewFile.mimetype : 'video/mp4'; return {name} @@ -94,13 +93,13 @@ function getVideoModal(previewFile) { playsInline={true} style={{maxWidth: '100%'}} > - + } -function getAudioModal(previewFile) { +function getAudioPreviewModal(previewFile) { const url = getMediaPathURL(previewFile); const path = previewFile.primary_path ?? previewFile.path; const name = path.replace(/^.*[\\\/]/, ''); @@ -123,7 +122,7 @@ function getAudioModal(previewFile) { } -function getSTLModal(previewFile) { +function getSTLPreviewModal(previewFile) { const url = getMediaPathURL(previewFile); const path = previewFile.primary_path ?? previewFile.path; const name = path.replace(/^.*[\\\/]/, ''); @@ -139,7 +138,7 @@ function getSTLModal(previewFile) { } -function getGenericModal(previewFile) { +function getGenericPreviewModal(previewFile) { // No special handler for this file type, just open it. const path = previewFile.primary_path ?? previewFile.path; const name = path.replace(/^.*[\\\/]/, ''); @@ -275,27 +274,27 @@ export function FilePreviewProvider({children}) { // Large text files should be downloaded. window.open(downloadURL); } else if (mimetype.startsWith('text/') || mimetype.startsWith('application/json')) { - setModalContent(getIframeModal(previewFile), url, downloadURL, path); + setModalContent(getIframePreviewModal(previewFile), url, downloadURL, path); } else if (mimetype.startsWith('video/')) { - setModalContent(getVideoModal(previewFile), url, downloadURL, path); + setModalContent(getVideoPreviewModal(previewFile), url, downloadURL, path); } else if (mimetype.startsWith('audio/')) { - setModalContent(getAudioModal(previewFile), url, downloadURL, path); + setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path); } else if (mimetype.startsWith('application/epub')) { const viewerURL = getEpubViewerURL(previewFile); - setModalContent(getEpubModal(previewFile), viewerURL, downloadURL, path); + setModalContent(getEpubPreviewModal(previewFile), viewerURL, downloadURL, path); } else if (mimetype.startsWith('application/pdf')) { - setModalContent(getIframeModal(previewFile), url, downloadURL, path); + setModalContent(getIframePreviewModal(previewFile), url, downloadURL, path); } else if (mimetype.startsWith('image/')) { - setModalContent(getImageModal(previewFile), url, null, path); + setModalContent(getImagePreviewModal(previewFile), url, null, path); } else if (mimetype.startsWith('model/stl')) { - setModalContent(getSTLModal(previewFile), null, downloadURL, path); + setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path); } else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.mp3')) { - setModalContent(getAudioModal(previewFile), url, downloadURL, path); + setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path); } else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.stl')) { - setModalContent(getSTLModal(previewFile), null, downloadURL, path); + setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path); } else { // No special handler for this file type, just open it. - setModalContent(getGenericModal(previewFile), url, downloadURL, path); + setModalContent(getGenericPreviewModal(previewFile), url, downloadURL, path); } } }, [previewFile]); diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 71cd5f58..8fffba35 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -193,11 +193,6 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { } let videoSource = ; - if (video.codec_names && video.codec_names.indexOf('mp4') >= 0) { - videoSource = - } else if (video.codec_names && video.codec_names.indexOf('vp9') >= 0) { - videoSource = - } return <> diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 86538472..b66b149e 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -77,30 +77,14 @@ class DownloadRow extends React.Component { super(props); this.state = { errorModalOpen: false, - deleteOpen: false, } } - openDelete = (e) => { - if (e) { - e.preventDefault(); - } - this.setState({deleteOpen: true}); - } - - closeDelete = (e) => { - if (e) { - e.preventDefault(); - } - this.setState({deleteOpen: false}); - } - handleDelete = async () => { const {id} = this.props; try { await deleteDownload(id); } finally { - this.closeDelete(); if (this.props.fetchDownloads) { this.props.fetchDownloads(); } @@ -109,7 +93,7 @@ class DownloadRow extends React.Component { render() { let {url, frequency, last_successful_download, status, location, next_download, error} = this.props; - const {errorModalOpen, deleteOpen} = this.state; + const {errorModalOpen} = this.state; const link = location ? (text) => {text} : @@ -171,9 +155,6 @@ class DownloadRow extends React.Component { class StoppableRow extends React.Component { constructor(props) { super(props); - this.state = { - errorModalOpen: false, - }; } handleDelete = async () => { @@ -194,7 +175,6 @@ class StoppableRow extends React.Component { render() { let {url, last_successful_download, status, location, error} = this.props; - let {errorModalOpen} = this.state; // Open downloads (/download), or external links in an anchor. const link = location && !location.startsWith('/download') ? @@ -326,13 +306,15 @@ export function RecurringDownloadsTable({downloads, fetchDownloads}) { export function DownloadsPage() { useTitle('Downloads'); - const {onceDownloads, recurringDownloads, fetchDownloads} = useDownloads(); + const {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads} = useDownloads(); + + const pendingOnceDownloadsSpan = pendingOnceDownloads > 0 ? ({pendingOnceDownloads}) : null; return <> -
Downloads
+
Downloads {pendingOnceDownloadsSpan}
Recurring Downloads
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 784566a3..2f531e12 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -551,12 +551,14 @@ export const useHotspot = () => { export const useDownloads = () => { const [onceDownloads, setOnceDownloads] = useState(null); const [recurringDownloads, setRecurringDownloads] = useState(null); + const [pendingOnceDownloads, setPendingOnceDownloads] = useState(null); const fetchDownloads = async () => { try { const data = await getDownloads(); setOnceDownloads(data['once_downloads']); setRecurringDownloads(data['recurring_downloads']); + setPendingOnceDownloads(data['pending_once_downloads']); } catch (e) { console.error(e); } @@ -564,7 +566,7 @@ export const useDownloads = () => { useRecurringTimeout(fetchDownloads, 3 * 1000); - return {onceDownloads, recurringDownloads, fetchDownloads} + return {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads} } export const useThrottle = () => { diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index 8cae8a31..45579412 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -44,7 +44,8 @@ async def video_modeler(): except Exception as e: if PYTEST: raise - logger.error(f'Unable to model Video {video_id=}', exc_info=e) + i = video.file_group.primary_path if video.file_group else video_id + logger.error(f'Unable to model Video {i=}', exc_info=e) file_group.indexed = True diff --git a/modules/videos/common.py b/modules/videos/common.py index 57a93f64..dbb10a69 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -18,6 +18,7 @@ from sqlalchemy.orm import Session from wrolpi.captions import FFMPEG_BIN from wrolpi.cmd import FFPROBE_BIN from wrolpi.common import logger, get_media_directory +from wrolpi.dates import seconds_to_timestamp from wrolpi.db import get_db_session, get_db_curs from wrolpi.vars import DEFAULT_FILE_PERMISSIONS from .errors import ChannelNameConflict, ChannelURLConflict, ChannelDirectoryConflict, ChannelSourceIdConflict @@ -118,24 +119,45 @@ def get_duration_from_ffmpeg(stderr: str) -> int: return duration -def generate_video_poster(video_path: Path) -> Tuple[Path, Optional[int]]: - """Create a poster (aka thumbnail) next to the provided video_path. Also returns the video duration.""" - poster_path = video_path.with_suffix('.jpg') - cmd = (FFMPEG_BIN, '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000', - str(poster_path)) - if poster_path.exists(): - # Poster already exists. - return poster_path, None +def ffmpeg_poster(video_path: Path, poster_path: Path, seconds: int): + """Use ffmpeg to create a poster file from a video file at a particular time of the video. + + Returns stderr so data about the command can be extracted.""" + timestamp = seconds_to_timestamp(seconds) + cmd = (FFMPEG_BIN, '-ss', timestamp, '-n', '-i', video_path, '-f', 'mjpeg', '-vframes', '1', poster_path) try: proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stderr = proc.stderr.decode() - logger.info(f'Generated poster at {poster_path}') except subprocess.CalledProcessError as e: - logger.warning(f'FFMPEG poster generation failed with stdout: {e.stdout.decode()}') - logger.warning(f'FFMPEG poster generation failed with stdout: {e.stderr.decode()}') + logger.error(f'Failed to create video poster', exc_info=e) raise - if not poster_path.exists(): - raise Exception(f'Failed to find generated poster! {poster_path}') + + return stderr + + +def ffmpeg_video_complete(video_path: Path, seconds: int = None) -> bool: + """Checks if video file is complete by taking screenshot from the end of the video.""" + seconds = seconds or get_video_duration(video_path) - 5 + with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as fh: + path = Path(fh.name) + path.unlink() + try: + ffmpeg_poster(video_path, path, seconds) + return True + except Exception: + return False + finally: + path.unlink() + + +def generate_video_poster(video_path: Path, seconds: int = 5) -> Tuple[Path, Optional[int]]: + """Create a poster (aka thumbnail) next to the provided video_path. Also returns the video duration.""" + poster_path = video_path.with_suffix('.jpg') + if poster_path.exists(): + # Poster already exists. + return poster_path, None + + stderr = ffmpeg_poster(video_path, poster_path, seconds) duration = get_duration_from_ffmpeg(stderr) return poster_path, duration diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 8554c5e7..a207a0f7 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -22,7 +22,7 @@ from wrolpi.errors import UnrecoverableDownloadError from wrolpi.files.lib import glob_shared_stem from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import get_no_channel_directory, get_videos_directory, update_view_counts +from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete from .errors import UnknownChannel, ChannelURLEmpty from .lib import get_downloader_config from .models import Video, Channel @@ -168,6 +168,9 @@ class VideoDownloader(Downloader, ABC): """Downloads a single video. Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`. + + Videos are validated in a few ways. A video file is valid when it has a complete video stream and an audio stream. + A complete video stream is validated by taking a screenshot at the end of the video. """ name = 'video' pretty_name = 'Videos' @@ -300,6 +303,12 @@ class VideoDownloader(Downloader, ABC): error=error, ) + if not ffmpeg_video_complete(video_path): + return DownloadResult( + success=False, + error='Video was incomplete', + ) + with get_db_session(commit=True) as session: # Find any files downloaded with the video (poster, caption, etc.). video_paths = glob_shared_stem(video_path) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index f9c429a7..966284c7 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -119,8 +119,11 @@ def validate_video(video: Video, channel_generate_poster: bool): if video_path and not video.duration: # Duration was not retrieved during poster generation. - if video.ffprobe_json and (video_streams := video.get_streams_by_codec_type('video')): - video.duration = float(video_streams[0]['duration']) + if video.ffprobe_json: + if duration := video.ffprobe_json['format'].get('duration'): + video.duration = float(duration) + elif video_streams := video.get_streams_by_codec_type('video'): + video.duration = float(video_streams[0]['duration']) else: # Slowest method. video.duration = get_video_duration(video_path) diff --git a/modules/videos/models.py b/modules/videos/models.py index 270fc71d..bc6b21b6 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -33,6 +33,7 @@ class Video(ModelHelper, Base): source_id = Column(String) upload_date: datetime.datetime = Column(TZDateTime) url = Column(String) + validated = Column(Boolean, default=False) view_count = Column(Integer) viewed = Column(TZDateTime) ffprobe_json = deferred(Column(JSON)) @@ -86,12 +87,13 @@ class Video(ModelHelper, Base): ) return d - def delete(self): + def delete(self, add_to_skip_list: bool = True): """Remove all files and File records related to this video. Delete this Video record. Add it to it's Channel's skip list.""" self.file_group.delete() - self.add_to_skip_list() + if add_to_skip_list: + self.add_to_skip_list() session = Session.object_session(self) session.delete(self) @@ -340,6 +342,10 @@ class Video(ModelHelper, Base): streams = [i for i in self.ffprobe_json['streams'] if i['codec_type'] == codec_type] return streams + def detect_is_complete(self): + from modules.videos.common import ffmpeg_video_complete + return ffmpeg_video_complete(self.video_path) + class Channel(ModelHelper, Base): __tablename__ = 'channel' diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index fb563c11..01d492a1 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -123,9 +123,10 @@ def test_generate_video_poster(video_file): A poster can be generated from a video file. """ poster_path = video_file.with_suffix('.jpg') - generate_video_poster(video_file) + _, duration = generate_video_poster(video_file) assert poster_path.is_file(), f'{poster_path} was not created!' assert poster_path.stat().st_size > 0 + assert duration == 5 def test_import_channel_downloads(test_session, channel_factory, test_channels_config): diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 57c23c82..cc12e725 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -319,7 +319,7 @@ def test_channel_download_no_download(test_session, video_download_manager, simp @pytest.mark.asyncio async def test_video_download(test_session, test_directory, simple_channel, video_download_manager, mock_video_process_runner, image_file): - """A video download is performed, files are associated.""" + """A video download is performed, files are grouped.""" simple_channel.source_id = example_video_json['channel_id'] simple_channel.directory = test_directory / 'videos/channel name' simple_channel.directory.mkdir(parents=True) diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages index a5a01d84..f669570a 100644 --- a/pi-gen/stage2/04-wrolpi/01-packages +++ b/pi-gen/stage2/04-wrolpi/01-packages @@ -20,6 +20,7 @@ gdal-bin git gnupg-agent hostapd +iotop kiwix kiwix-tools libagg-dev @@ -74,6 +75,7 @@ python3-venv renderd software-properties-common tar +tmux ttf-unifont unzip vim diff --git a/scripts/find_corrupt_videos.py b/scripts/find_corrupt_videos.py index f46617d8..8ec16a3f 100755 --- a/scripts/find_corrupt_videos.py +++ b/scripts/find_corrupt_videos.py @@ -7,37 +7,28 @@ will be prompted to delete it. import argparse import asyncio import logging +import pathlib +import subprocess import sys +import tempfile +from http import HTTPStatus from typing import List +import aiohttp +from sqlalchemy.orm import Session from yt_dlp.utils import YoutubeDLError -from modules.videos.downloader import VideoDownloader, extract_info -from modules.videos.models import Video +from modules.videos.downloader import extract_info, VideoDownloader +from modules.videos.models import Video, Channel +from wrolpi.dates import seconds_to_timestamp from wrolpi.db import get_db_session, get_db_curs -from wrolpi.downloader import download_manager logger = logging.getLogger() -def get_total_corrupted_videos() -> int: - with get_db_curs() as curs: - stmt = ''' - WITH all_videos AS ( - select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types - from video v - cross join lateral json_array_elements(ffprobe_json -> 'streams') s - group by v.id) - SELECT COUNT(all_videos.id) - FROM all_videos - WHERE NOT all_videos.codec_types @> '{audio,video}'::TEXT[] - ''' - curs.execute(stmt) - return int(curs.fetchone()[0]) - - -def get_corrupt_videos(limit: int, offset: int) -> List[dict]: - params = dict(limit=limit, offset=offset) +def get_incomplete_videos(limit: int, offset: int, channel_id: int) -> List[dict]: + """Returns Video records that do not have both video and audio streams.""" + params = dict(limit=limit, offset=offset, channel_id=channel_id) with get_db_curs() as curs: stmt = ''' WITH all_videos AS ( @@ -46,6 +37,7 @@ def get_corrupt_videos(limit: int, offset: int) -> List[dict]: select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types from video v cross join lateral json_array_elements(ffprobe_json -> 'streams') s + where channel_id = %(channel_id)s group by v.id) SELECT all_videos.id AS video_id, fg.primary_path, all_videos.codec_types FROM all_videos @@ -62,16 +54,26 @@ def get_corrupt_videos(limit: int, offset: int) -> List[dict]: return [dict(i) for i in curs.fetchall()] -def iterate_corrupt_videos() -> List[dict]: +def iterate_incomplete_videos(channel_id: int = None) -> List[dict]: limit = 20 offset = 0 - while True: - videos = get_corrupt_videos(limit, offset) - offset += limit - if len(videos) < limit: - # Ran out of corrupt videos. - break - yield videos + with get_db_session() as session: + channel_ids = [channel_id, ] if channel_id else [int(i[0]) for i in session.query(Channel.id)] + found = False + for channel_id in channel_ids: + channel_id: int + while True: + videos = get_incomplete_videos(limit, offset, channel_id) + offset += limit + if videos: + yield videos + if len(videos) < limit: + # Ran out of corrupt videos. + if found: + logger.debug(f'Ran out of corrupt videos for {channel_id=}') + else: + logger.debug(f'No corrupt videos for {channel_id=}') + break def confirm(msg: str) -> bool: @@ -86,47 +88,103 @@ def confirm(msg: str) -> bool: return False -async def find_corrupt_videos(): - total_corrupted = get_total_corrupted_videos() - count = 0 +def can_be_downloaded(url: str) -> bool: + try: + extract_info(url) + return True + except YoutubeDLError: + return False + +async def create_download(url: str, downloader: str): + json_ = {'urls': url, 'downloader': downloader} + async with aiohttp.ClientSession() as session: + async with session.post('http://127.0.0.1:8080/api/download', json=json_) as response: + if response.status == HTTPStatus.NO_CONTENT: + logger.info(f'Created download for {url}') + else: + raise Exception(f'Failed to create download for {url}') + + +async def download_or_ask_delete(video: Video, session: Session, delete_message: str): + if video.url and can_be_downloaded(video.url): + await create_download(video.url, VideoDownloader.name) + # Always re-download when possible. + do_delete = True + else: + logger.error(f'Video cannot be downloaded: {video.url or video}') + do_delete = confirm(delete_message) + + if do_delete: + logger.debug(f'Deleting {video}') + video.delete(add_to_skip_list=False) + session.commit() + + +async def find_corrupt_videos(channel_id: int = None): with get_db_session() as session: - for chunk in iterate_corrupt_videos(): + for chunk in iterate_incomplete_videos(channel_id=channel_id): video_ids = [i['video_id'] for i in chunk] videos = session.query(Video).filter(Video.id.in_(video_ids)).all() for video in videos: - print() - print(f'Video (remaining: {total_corrupted - count}): {video}') - count += 1 - streams = (await video.get_ffprobe_json())['streams'] - codec_names = [i['codec_name'] for i in streams] - print(f'codecs: {codec_names}') + path = video.video_path + await download_or_ask_delete(video, session, f'Missing audio. {path} Delete? (y/N)') - url = video.url - if url: - try: - extract_info(url) - print(f'Re-downloading') - download_manager.create_download(url, VideoDownloader.name, session=session) - video.delete() + with get_db_session() as session: + videos = session.query(Video).filter(Video.validated == False) # noqa + if channel_id: + videos = videos.filter(Video.channel_id == channel_id) + for video in videos: + if not video.duration: + logger.warning(f'No duration for {video}') + continue + if video.duration > 5: + timestamp = seconds_to_timestamp(video.duration - 5) + else: + timestamp = seconds_to_timestamp(video.duration - 1) + # Create a screenshot from the end of the video. If this fails, the video is corrupt. + with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as fh: + fh = pathlib.Path(fh.name) + fh.unlink() + try: + cmd = ( + 'ffmpeg', '-n', + '-ss', timestamp, + '-i', str(video.video_path), + '-f', 'mjpeg', + '-vframes', '1', + '-q:v', '2', + str(fh), + ) + proc = subprocess.run(cmd, check=True, capture_output=True, timeout=10) + if fh.is_file() and fh.stat().st_size: + logger.debug(f'{video.video_path.name} is valid') + video.validated = True session.commit() + fh.unlink() continue - except YoutubeDLError: - logger.error(f'Video cannot be downloaded: {url}') - if confirm(f'Delete? (y/N)'): - video.delete() - session.commit() - continue + except subprocess.CalledProcessError as e: + logger.error(f'Failed to generate poster for {video.video_path}', exc_info=e) - if confirm('Delete? (y/N)'): - video.delete() - session.commit() + path = video.video_path + await download_or_ask_delete(video, session, f'Corrupt video. {path} Delete? (y/N)') if __name__ == '__main__': parser = argparse.ArgumentParser() + parser.add_argument('-v', action='count') + parser.add_argument('-c', '--channel-id', type=int, + help='The id of the channel to search') args = parser.parse_args() + if args.v == 0: + logger.setLevel(logging.WARNING) + elif args.v == 1: + logger.setLevel(logging.INFO) + elif args.v >= 2: + logger.setLevel(logging.DEBUG) + logger.debug(f'Debug logging') + loop = asyncio.get_event_loop() - coro = find_corrupt_videos() + coro = find_corrupt_videos(channel_id=args.channel_id) loop.run_until_complete(coro) diff --git a/scripts/install_debian_11.sh b/scripts/install_debian_11.sh index e33738a4..db3abba4 100755 --- a/scripts/install_debian_11.sh +++ b/scripts/install_debian_11.sh @@ -20,6 +20,7 @@ apt-get install -y \ gnupg-agent \ hostapd \ htop \ + iotop \ kiwix \ kiwix-tools \ libpq-dev \ @@ -34,6 +35,7 @@ apt-get install -y \ python3-pip \ python3-venv \ software-properties-common \ + tmux \ vim \ zim-tools diff --git a/scripts/install_raspberrypios.sh b/scripts/install_raspberrypios.sh index 829d48d5..07e2df7a 100755 --- a/scripts/install_raspberrypios.sh +++ b/scripts/install_raspberrypios.sh @@ -20,6 +20,7 @@ apt-get install -y \ gnupg-agent \ hostapd \ htop \ + iotop \ kiwix \ kiwix-tools \ libpq-dev \ @@ -34,6 +35,7 @@ apt-get install -y \ python3-pip \ python3-venv \ software-properties-common \ + tmux \ vim \ zim-tools diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 7456b721..05187c43 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -886,9 +886,22 @@ class DownloadManager: curs.execute(stmt) once_downloads = list(map(dict, curs.fetchall())) + stmt = ''' + SELECT + COUNT(id) FILTER ( + WHERE frequency IS NULL + AND (status = 'pending' OR status = 'new') + ) + FROM + download + ''' + curs.execute(stmt) + pending_once_downloads = curs.fetchone()[0] + data = dict( recurring_downloads=recurring_downloads, once_downloads=once_downloads, + pending_once_downloads=pending_once_downloads, ) return data