mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Creating Video.validated which is used by find_corrupt_videos.py to search.
This commit is contained in:
parent
d0d315cd67
commit
e8e5b5727f
17 changed files with 253 additions and 125 deletions
31
alembic/versions/f360a276cdcf_video_valid.py
Normal file
31
alembic/versions/f360a276cdcf_video_valid.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""Video.valid.
|
||||
|
||||
Revision ID: f360a276cdcf
|
||||
Revises: 8bdb48a486b9
|
||||
Create Date: 2023-08-07 22:49:37.001794
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'f360a276cdcf'
|
||||
down_revision = '8bdb48a486b9'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS validated')
|
||||
|
|
@ -33,7 +33,7 @@ function getEpubViewerURL(previewFile) {
|
|||
}
|
||||
}
|
||||
|
||||
function getIframeModal(previewFile) {
|
||||
function getIframePreviewModal(previewFile) {
|
||||
const url = getMediaPathURL(previewFile);
|
||||
return <ModalContent>
|
||||
<div className='full-height'>
|
||||
|
|
@ -47,7 +47,7 @@ function getIframeModal(previewFile) {
|
|||
</ModalContent>
|
||||
}
|
||||
|
||||
function getImageModal(previewFile) {
|
||||
function getImagePreviewModal(previewFile) {
|
||||
const path = previewFile.primary_path ?? previewFile.path;
|
||||
const url = getMediaPathURL(previewFile);
|
||||
// Get the file name from the absolute path.
|
||||
|
|
@ -64,7 +64,7 @@ function getImageModal(previewFile) {
|
|||
</React.Fragment>
|
||||
}
|
||||
|
||||
function getEpubModal(previewFile) {
|
||||
function getEpubPreviewModal(previewFile) {
|
||||
const downloadURL = getDownloadPathURL(previewFile);
|
||||
const viewerUrl = `/epub/epub.html?url=${downloadURL}`;
|
||||
return <ModalContent>
|
||||
|
|
@ -77,11 +77,10 @@ function getEpubModal(previewFile) {
|
|||
</ModalContent>
|
||||
}
|
||||
|
||||
function getVideoModal(previewFile) {
|
||||
function getVideoPreviewModal(previewFile) {
|
||||
const url = getMediaPathURL(previewFile);
|
||||
const path = previewFile.primary_path ?? previewFile.path;
|
||||
const name = path.replace(/^.*[\\\/]/, '');
|
||||
const type = previewFile.mimetype ? previewFile.mimetype : 'video/mp4';
|
||||
return <React.Fragment>
|
||||
<ModalHeader>
|
||||
{name}
|
||||
|
|
@ -94,13 +93,13 @@ function getVideoModal(previewFile) {
|
|||
playsInline={true}
|
||||
style={{maxWidth: '100%'}}
|
||||
>
|
||||
<source src={url} type={type}/>
|
||||
<source src={url}/>
|
||||
</video>
|
||||
</ModalContent>
|
||||
</React.Fragment>
|
||||
}
|
||||
|
||||
function getAudioModal(previewFile) {
|
||||
function getAudioPreviewModal(previewFile) {
|
||||
const url = getMediaPathURL(previewFile);
|
||||
const path = previewFile.primary_path ?? previewFile.path;
|
||||
const name = path.replace(/^.*[\\\/]/, '');
|
||||
|
|
@ -123,7 +122,7 @@ function getAudioModal(previewFile) {
|
|||
</React.Fragment>
|
||||
}
|
||||
|
||||
function getSTLModal(previewFile) {
|
||||
function getSTLPreviewModal(previewFile) {
|
||||
const url = getMediaPathURL(previewFile);
|
||||
const path = previewFile.primary_path ?? previewFile.path;
|
||||
const name = path.replace(/^.*[\\\/]/, '');
|
||||
|
|
@ -139,7 +138,7 @@ function getSTLModal(previewFile) {
|
|||
</React.Fragment>
|
||||
}
|
||||
|
||||
function getGenericModal(previewFile) {
|
||||
function getGenericPreviewModal(previewFile) {
|
||||
// No special handler for this file type, just open it.
|
||||
const path = previewFile.primary_path ?? previewFile.path;
|
||||
const name = path.replace(/^.*[\\\/]/, '');
|
||||
|
|
@ -275,27 +274,27 @@ export function FilePreviewProvider({children}) {
|
|||
// Large text files should be downloaded.
|
||||
window.open(downloadURL);
|
||||
} else if (mimetype.startsWith('text/') || mimetype.startsWith('application/json')) {
|
||||
setModalContent(getIframeModal(previewFile), url, downloadURL, path);
|
||||
setModalContent(getIframePreviewModal(previewFile), url, downloadURL, path);
|
||||
} else if (mimetype.startsWith('video/')) {
|
||||
setModalContent(getVideoModal(previewFile), url, downloadURL, path);
|
||||
setModalContent(getVideoPreviewModal(previewFile), url, downloadURL, path);
|
||||
} else if (mimetype.startsWith('audio/')) {
|
||||
setModalContent(getAudioModal(previewFile), url, downloadURL, path);
|
||||
setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path);
|
||||
} else if (mimetype.startsWith('application/epub')) {
|
||||
const viewerURL = getEpubViewerURL(previewFile);
|
||||
setModalContent(getEpubModal(previewFile), viewerURL, downloadURL, path);
|
||||
setModalContent(getEpubPreviewModal(previewFile), viewerURL, downloadURL, path);
|
||||
} else if (mimetype.startsWith('application/pdf')) {
|
||||
setModalContent(getIframeModal(previewFile), url, downloadURL, path);
|
||||
setModalContent(getIframePreviewModal(previewFile), url, downloadURL, path);
|
||||
} else if (mimetype.startsWith('image/')) {
|
||||
setModalContent(getImageModal(previewFile), url, null, path);
|
||||
setModalContent(getImagePreviewModal(previewFile), url, null, path);
|
||||
} else if (mimetype.startsWith('model/stl')) {
|
||||
setModalContent(getSTLModal(previewFile), null, downloadURL, path);
|
||||
setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path);
|
||||
} else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.mp3')) {
|
||||
setModalContent(getAudioModal(previewFile), url, downloadURL, path);
|
||||
setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path);
|
||||
} else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.stl')) {
|
||||
setModalContent(getSTLModal(previewFile), null, downloadURL, path);
|
||||
setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path);
|
||||
} else {
|
||||
// No special handler for this file type, just open it.
|
||||
setModalContent(getGenericModal(previewFile), url, downloadURL, path);
|
||||
setModalContent(getGenericPreviewModal(previewFile), url, downloadURL, path);
|
||||
}
|
||||
}
|
||||
}, [previewFile]);
|
||||
|
|
|
|||
|
|
@ -193,11 +193,6 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
}
|
||||
|
||||
let videoSource = <source src={videoUrl}/>;
|
||||
if (video.codec_names && video.codec_names.indexOf('mp4') >= 0) {
|
||||
videoSource = <source src={videoUrl} type="video/mp4"/>
|
||||
} else if (video.codec_names && video.codec_names.indexOf('vp9') >= 0) {
|
||||
videoSource = <source src={videoUrl} type='video/mp4;codecs="vp9, vorbis"'/>
|
||||
}
|
||||
|
||||
return <>
|
||||
<Container style={{margin: '1em'}}>
|
||||
|
|
|
|||
|
|
@ -77,30 +77,14 @@ class DownloadRow extends React.Component {
|
|||
super(props);
|
||||
this.state = {
|
||||
errorModalOpen: false,
|
||||
deleteOpen: false,
|
||||
}
|
||||
}
|
||||
|
||||
openDelete = (e) => {
|
||||
if (e) {
|
||||
e.preventDefault();
|
||||
}
|
||||
this.setState({deleteOpen: true});
|
||||
}
|
||||
|
||||
closeDelete = (e) => {
|
||||
if (e) {
|
||||
e.preventDefault();
|
||||
}
|
||||
this.setState({deleteOpen: false});
|
||||
}
|
||||
|
||||
handleDelete = async () => {
|
||||
const {id} = this.props;
|
||||
try {
|
||||
await deleteDownload(id);
|
||||
} finally {
|
||||
this.closeDelete();
|
||||
if (this.props.fetchDownloads) {
|
||||
this.props.fetchDownloads();
|
||||
}
|
||||
|
|
@ -109,7 +93,7 @@ class DownloadRow extends React.Component {
|
|||
|
||||
render() {
|
||||
let {url, frequency, last_successful_download, status, location, next_download, error} = this.props;
|
||||
const {errorModalOpen, deleteOpen} = this.state;
|
||||
const {errorModalOpen} = this.state;
|
||||
|
||||
const link = location ?
|
||||
(text) => <Link to={location}>{text}</Link> :
|
||||
|
|
@ -171,9 +155,6 @@ class DownloadRow extends React.Component {
|
|||
class StoppableRow extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
errorModalOpen: false,
|
||||
};
|
||||
}
|
||||
|
||||
handleDelete = async () => {
|
||||
|
|
@ -194,7 +175,6 @@ class StoppableRow extends React.Component {
|
|||
|
||||
render() {
|
||||
let {url, last_successful_download, status, location, error} = this.props;
|
||||
let {errorModalOpen} = this.state;
|
||||
|
||||
// Open downloads (/download), or external links in an anchor.
|
||||
const link = location && !location.startsWith('/download') ?
|
||||
|
|
@ -326,13 +306,15 @@ export function RecurringDownloadsTable({downloads, fetchDownloads}) {
|
|||
export function DownloadsPage() {
|
||||
useTitle('Downloads');
|
||||
|
||||
const {onceDownloads, recurringDownloads, fetchDownloads} = useDownloads();
|
||||
const {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads} = useDownloads();
|
||||
|
||||
const pendingOnceDownloadsSpan = pendingOnceDownloads > 0 ? <span>({pendingOnceDownloads})</span> : null;
|
||||
|
||||
return <>
|
||||
<WROLModeMessage content='Downloads are disabled because WROL Mode is enabled.'/>
|
||||
<DisableDownloadsToggle/>
|
||||
|
||||
<Header as='h1'>Downloads</Header>
|
||||
<Header as='h1'>Downloads {pendingOnceDownloadsSpan}</Header>
|
||||
<OnceDownloadsTable downloads={onceDownloads} fetchDownloads={fetchDownloads}/>
|
||||
|
||||
<Header as='h1'>Recurring Downloads</Header>
|
||||
|
|
|
|||
|
|
@ -551,12 +551,14 @@ export const useHotspot = () => {
|
|||
export const useDownloads = () => {
|
||||
const [onceDownloads, setOnceDownloads] = useState(null);
|
||||
const [recurringDownloads, setRecurringDownloads] = useState(null);
|
||||
const [pendingOnceDownloads, setPendingOnceDownloads] = useState(null);
|
||||
|
||||
const fetchDownloads = async () => {
|
||||
try {
|
||||
const data = await getDownloads();
|
||||
setOnceDownloads(data['once_downloads']);
|
||||
setRecurringDownloads(data['recurring_downloads']);
|
||||
setPendingOnceDownloads(data['pending_once_downloads']);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
|
|
@ -564,7 +566,7 @@ export const useDownloads = () => {
|
|||
|
||||
useRecurringTimeout(fetchDownloads, 3 * 1000);
|
||||
|
||||
return {onceDownloads, recurringDownloads, fetchDownloads}
|
||||
return {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads}
|
||||
}
|
||||
|
||||
export const useThrottle = () => {
|
||||
|
|
|
|||
|
|
@ -44,7 +44,8 @@ async def video_modeler():
|
|||
except Exception as e:
|
||||
if PYTEST:
|
||||
raise
|
||||
logger.error(f'Unable to model Video {video_id=}', exc_info=e)
|
||||
i = video.file_group.primary_path if video.file_group else video_id
|
||||
logger.error(f'Unable to model Video {i=}', exc_info=e)
|
||||
|
||||
file_group.indexed = True
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from sqlalchemy.orm import Session
|
|||
from wrolpi.captions import FFMPEG_BIN
|
||||
from wrolpi.cmd import FFPROBE_BIN
|
||||
from wrolpi.common import logger, get_media_directory
|
||||
from wrolpi.dates import seconds_to_timestamp
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
|
||||
from .errors import ChannelNameConflict, ChannelURLConflict, ChannelDirectoryConflict, ChannelSourceIdConflict
|
||||
|
|
@ -118,24 +119,45 @@ def get_duration_from_ffmpeg(stderr: str) -> int:
|
|||
return duration
|
||||
|
||||
|
||||
def generate_video_poster(video_path: Path) -> Tuple[Path, Optional[int]]:
|
||||
"""Create a poster (aka thumbnail) next to the provided video_path. Also returns the video duration."""
|
||||
poster_path = video_path.with_suffix('.jpg')
|
||||
cmd = (FFMPEG_BIN, '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000',
|
||||
str(poster_path))
|
||||
if poster_path.exists():
|
||||
# Poster already exists.
|
||||
return poster_path, None
|
||||
def ffmpeg_poster(video_path: Path, poster_path: Path, seconds: int):
|
||||
"""Use ffmpeg to create a poster file from a video file at a particular time of the video.
|
||||
|
||||
Returns stderr so data about the command can be extracted."""
|
||||
timestamp = seconds_to_timestamp(seconds)
|
||||
cmd = (FFMPEG_BIN, '-ss', timestamp, '-n', '-i', video_path, '-f', 'mjpeg', '-vframes', '1', poster_path)
|
||||
try:
|
||||
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stderr = proc.stderr.decode()
|
||||
logger.info(f'Generated poster at {poster_path}')
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f'FFMPEG poster generation failed with stdout: {e.stdout.decode()}')
|
||||
logger.warning(f'FFMPEG poster generation failed with stdout: {e.stderr.decode()}')
|
||||
logger.error(f'Failed to create video poster', exc_info=e)
|
||||
raise
|
||||
if not poster_path.exists():
|
||||
raise Exception(f'Failed to find generated poster! {poster_path}')
|
||||
|
||||
return stderr
|
||||
|
||||
|
||||
def ffmpeg_video_complete(video_path: Path, seconds: int = None) -> bool:
|
||||
"""Checks if video file is complete by taking screenshot from the end of the video."""
|
||||
seconds = seconds or get_video_duration(video_path) - 5
|
||||
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as fh:
|
||||
path = Path(fh.name)
|
||||
path.unlink()
|
||||
try:
|
||||
ffmpeg_poster(video_path, path, seconds)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
path.unlink()
|
||||
|
||||
|
||||
def generate_video_poster(video_path: Path, seconds: int = 5) -> Tuple[Path, Optional[int]]:
|
||||
"""Create a poster (aka thumbnail) next to the provided video_path. Also returns the video duration."""
|
||||
poster_path = video_path.with_suffix('.jpg')
|
||||
if poster_path.exists():
|
||||
# Poster already exists.
|
||||
return poster_path, None
|
||||
|
||||
stderr = ffmpeg_poster(video_path, poster_path, seconds)
|
||||
|
||||
duration = get_duration_from_ffmpeg(stderr)
|
||||
return poster_path, duration
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from wrolpi.errors import UnrecoverableDownloadError
|
|||
from wrolpi.files.lib import glob_shared_stem
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import get_no_channel_directory, get_videos_directory, update_view_counts
|
||||
from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete
|
||||
from .errors import UnknownChannel, ChannelURLEmpty
|
||||
from .lib import get_downloader_config
|
||||
from .models import Video, Channel
|
||||
|
|
@ -168,6 +168,9 @@ class VideoDownloader(Downloader, ABC):
|
|||
"""Downloads a single video.
|
||||
|
||||
Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`.
|
||||
|
||||
Videos are validated in a few ways. A video file is valid when it has a complete video stream and an audio stream.
|
||||
A complete video stream is validated by taking a screenshot at the end of the video.
|
||||
"""
|
||||
name = 'video'
|
||||
pretty_name = 'Videos'
|
||||
|
|
@ -300,6 +303,12 @@ class VideoDownloader(Downloader, ABC):
|
|||
error=error,
|
||||
)
|
||||
|
||||
if not ffmpeg_video_complete(video_path):
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Video was incomplete',
|
||||
)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Find any files downloaded with the video (poster, caption, etc.).
|
||||
video_paths = glob_shared_stem(video_path)
|
||||
|
|
|
|||
|
|
@ -119,8 +119,11 @@ def validate_video(video: Video, channel_generate_poster: bool):
|
|||
|
||||
if video_path and not video.duration:
|
||||
# Duration was not retrieved during poster generation.
|
||||
if video.ffprobe_json and (video_streams := video.get_streams_by_codec_type('video')):
|
||||
video.duration = float(video_streams[0]['duration'])
|
||||
if video.ffprobe_json:
|
||||
if duration := video.ffprobe_json['format'].get('duration'):
|
||||
video.duration = float(duration)
|
||||
elif video_streams := video.get_streams_by_codec_type('video'):
|
||||
video.duration = float(video_streams[0]['duration'])
|
||||
else:
|
||||
# Slowest method.
|
||||
video.duration = get_video_duration(video_path)
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ class Video(ModelHelper, Base):
|
|||
source_id = Column(String)
|
||||
upload_date: datetime.datetime = Column(TZDateTime)
|
||||
url = Column(String)
|
||||
validated = Column(Boolean, default=False)
|
||||
view_count = Column(Integer)
|
||||
viewed = Column(TZDateTime)
|
||||
ffprobe_json = deferred(Column(JSON))
|
||||
|
|
@ -86,12 +87,13 @@ class Video(ModelHelper, Base):
|
|||
)
|
||||
return d
|
||||
|
||||
def delete(self):
|
||||
def delete(self, add_to_skip_list: bool = True):
|
||||
"""Remove all files and File records related to this video. Delete this Video record.
|
||||
Add it to it's Channel's skip list."""
|
||||
self.file_group.delete()
|
||||
|
||||
self.add_to_skip_list()
|
||||
if add_to_skip_list:
|
||||
self.add_to_skip_list()
|
||||
session = Session.object_session(self)
|
||||
session.delete(self)
|
||||
|
||||
|
|
@ -340,6 +342,10 @@ class Video(ModelHelper, Base):
|
|||
streams = [i for i in self.ffprobe_json['streams'] if i['codec_type'] == codec_type]
|
||||
return streams
|
||||
|
||||
def detect_is_complete(self):
|
||||
from modules.videos.common import ffmpeg_video_complete
|
||||
return ffmpeg_video_complete(self.video_path)
|
||||
|
||||
|
||||
class Channel(ModelHelper, Base):
|
||||
__tablename__ = 'channel'
|
||||
|
|
|
|||
|
|
@ -123,9 +123,10 @@ def test_generate_video_poster(video_file):
|
|||
A poster can be generated from a video file.
|
||||
"""
|
||||
poster_path = video_file.with_suffix('.jpg')
|
||||
generate_video_poster(video_file)
|
||||
_, duration = generate_video_poster(video_file)
|
||||
assert poster_path.is_file(), f'{poster_path} was not created!'
|
||||
assert poster_path.stat().st_size > 0
|
||||
assert duration == 5
|
||||
|
||||
|
||||
def test_import_channel_downloads(test_session, channel_factory, test_channels_config):
|
||||
|
|
|
|||
|
|
@ -319,7 +319,7 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
|
|||
@pytest.mark.asyncio
|
||||
async def test_video_download(test_session, test_directory, simple_channel, video_download_manager,
|
||||
mock_video_process_runner, image_file):
|
||||
"""A video download is performed, files are associated."""
|
||||
"""A video download is performed, files are grouped."""
|
||||
simple_channel.source_id = example_video_json['channel_id']
|
||||
simple_channel.directory = test_directory / 'videos/channel name'
|
||||
simple_channel.directory.mkdir(parents=True)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ gdal-bin
|
|||
git
|
||||
gnupg-agent
|
||||
hostapd
|
||||
iotop
|
||||
kiwix
|
||||
kiwix-tools
|
||||
libagg-dev
|
||||
|
|
@ -74,6 +75,7 @@ python3-venv
|
|||
renderd
|
||||
software-properties-common
|
||||
tar
|
||||
tmux
|
||||
ttf-unifont
|
||||
unzip
|
||||
vim
|
||||
|
|
|
|||
|
|
@ -7,37 +7,28 @@ will be prompted to delete it.
|
|||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from http import HTTPStatus
|
||||
from typing import List
|
||||
|
||||
import aiohttp
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp.utils import YoutubeDLError
|
||||
|
||||
from modules.videos.downloader import VideoDownloader, extract_info
|
||||
from modules.videos.models import Video
|
||||
from modules.videos.downloader import extract_info, VideoDownloader
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.dates import seconds_to_timestamp
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import download_manager
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
def get_total_corrupted_videos() -> int:
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
WITH all_videos AS (
|
||||
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
|
||||
from video v
|
||||
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
|
||||
group by v.id)
|
||||
SELECT COUNT(all_videos.id)
|
||||
FROM all_videos
|
||||
WHERE NOT all_videos.codec_types @> '{audio,video}'::TEXT[]
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
return int(curs.fetchone()[0])
|
||||
|
||||
|
||||
def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
|
||||
params = dict(limit=limit, offset=offset)
|
||||
def get_incomplete_videos(limit: int, offset: int, channel_id: int) -> List[dict]:
|
||||
"""Returns Video records that do not have both video and audio streams."""
|
||||
params = dict(limit=limit, offset=offset, channel_id=channel_id)
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
WITH all_videos AS (
|
||||
|
|
@ -46,6 +37,7 @@ def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
|
|||
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
|
||||
from video v
|
||||
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
|
||||
where channel_id = %(channel_id)s
|
||||
group by v.id)
|
||||
SELECT all_videos.id AS video_id, fg.primary_path, all_videos.codec_types
|
||||
FROM all_videos
|
||||
|
|
@ -62,16 +54,26 @@ def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
|
|||
return [dict(i) for i in curs.fetchall()]
|
||||
|
||||
|
||||
def iterate_corrupt_videos() -> List[dict]:
|
||||
def iterate_incomplete_videos(channel_id: int = None) -> List[dict]:
|
||||
limit = 20
|
||||
offset = 0
|
||||
while True:
|
||||
videos = get_corrupt_videos(limit, offset)
|
||||
offset += limit
|
||||
if len(videos) < limit:
|
||||
# Ran out of corrupt videos.
|
||||
break
|
||||
yield videos
|
||||
with get_db_session() as session:
|
||||
channel_ids = [channel_id, ] if channel_id else [int(i[0]) for i in session.query(Channel.id)]
|
||||
found = False
|
||||
for channel_id in channel_ids:
|
||||
channel_id: int
|
||||
while True:
|
||||
videos = get_incomplete_videos(limit, offset, channel_id)
|
||||
offset += limit
|
||||
if videos:
|
||||
yield videos
|
||||
if len(videos) < limit:
|
||||
# Ran out of corrupt videos.
|
||||
if found:
|
||||
logger.debug(f'Ran out of corrupt videos for {channel_id=}')
|
||||
else:
|
||||
logger.debug(f'No corrupt videos for {channel_id=}')
|
||||
break
|
||||
|
||||
|
||||
def confirm(msg: str) -> bool:
|
||||
|
|
@ -86,47 +88,103 @@ def confirm(msg: str) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
async def find_corrupt_videos():
|
||||
total_corrupted = get_total_corrupted_videos()
|
||||
count = 0
|
||||
def can_be_downloaded(url: str) -> bool:
|
||||
try:
|
||||
extract_info(url)
|
||||
return True
|
||||
except YoutubeDLError:
|
||||
return False
|
||||
|
||||
|
||||
async def create_download(url: str, downloader: str):
|
||||
json_ = {'urls': url, 'downloader': downloader}
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post('http://127.0.0.1:8080/api/download', json=json_) as response:
|
||||
if response.status == HTTPStatus.NO_CONTENT:
|
||||
logger.info(f'Created download for {url}')
|
||||
else:
|
||||
raise Exception(f'Failed to create download for {url}')
|
||||
|
||||
|
||||
async def download_or_ask_delete(video: Video, session: Session, delete_message: str):
|
||||
if video.url and can_be_downloaded(video.url):
|
||||
await create_download(video.url, VideoDownloader.name)
|
||||
# Always re-download when possible.
|
||||
do_delete = True
|
||||
else:
|
||||
logger.error(f'Video cannot be downloaded: {video.url or video}')
|
||||
do_delete = confirm(delete_message)
|
||||
|
||||
if do_delete:
|
||||
logger.debug(f'Deleting {video}')
|
||||
video.delete(add_to_skip_list=False)
|
||||
session.commit()
|
||||
|
||||
|
||||
async def find_corrupt_videos(channel_id: int = None):
|
||||
with get_db_session() as session:
|
||||
for chunk in iterate_corrupt_videos():
|
||||
for chunk in iterate_incomplete_videos(channel_id=channel_id):
|
||||
video_ids = [i['video_id'] for i in chunk]
|
||||
videos = session.query(Video).filter(Video.id.in_(video_ids)).all()
|
||||
for video in videos:
|
||||
print()
|
||||
print(f'Video (remaining: {total_corrupted - count}): {video}')
|
||||
count += 1
|
||||
streams = (await video.get_ffprobe_json())['streams']
|
||||
codec_names = [i['codec_name'] for i in streams]
|
||||
print(f'codecs: {codec_names}')
|
||||
path = video.video_path
|
||||
await download_or_ask_delete(video, session, f'Missing audio. {path} Delete? (y/N)')
|
||||
|
||||
url = video.url
|
||||
if url:
|
||||
try:
|
||||
extract_info(url)
|
||||
print(f'Re-downloading')
|
||||
download_manager.create_download(url, VideoDownloader.name, session=session)
|
||||
video.delete()
|
||||
with get_db_session() as session:
|
||||
videos = session.query(Video).filter(Video.validated == False) # noqa
|
||||
if channel_id:
|
||||
videos = videos.filter(Video.channel_id == channel_id)
|
||||
for video in videos:
|
||||
if not video.duration:
|
||||
logger.warning(f'No duration for {video}')
|
||||
continue
|
||||
if video.duration > 5:
|
||||
timestamp = seconds_to_timestamp(video.duration - 5)
|
||||
else:
|
||||
timestamp = seconds_to_timestamp(video.duration - 1)
|
||||
# Create a screenshot from the end of the video. If this fails, the video is corrupt.
|
||||
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as fh:
|
||||
fh = pathlib.Path(fh.name)
|
||||
fh.unlink()
|
||||
try:
|
||||
cmd = (
|
||||
'ffmpeg', '-n',
|
||||
'-ss', timestamp,
|
||||
'-i', str(video.video_path),
|
||||
'-f', 'mjpeg',
|
||||
'-vframes', '1',
|
||||
'-q:v', '2',
|
||||
str(fh),
|
||||
)
|
||||
proc = subprocess.run(cmd, check=True, capture_output=True, timeout=10)
|
||||
if fh.is_file() and fh.stat().st_size:
|
||||
logger.debug(f'{video.video_path.name} is valid')
|
||||
video.validated = True
|
||||
session.commit()
|
||||
fh.unlink()
|
||||
continue
|
||||
except YoutubeDLError:
|
||||
logger.error(f'Video cannot be downloaded: {url}')
|
||||
if confirm(f'Delete? (y/N)'):
|
||||
video.delete()
|
||||
session.commit()
|
||||
continue
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f'Failed to generate poster for {video.video_path}', exc_info=e)
|
||||
|
||||
if confirm('Delete? (y/N)'):
|
||||
video.delete()
|
||||
session.commit()
|
||||
path = video.video_path
|
||||
await download_or_ask_delete(video, session, f'Corrupt video. {path} Delete? (y/N)')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-v', action='count')
|
||||
parser.add_argument('-c', '--channel-id', type=int,
|
||||
help='The id of the channel to search')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.v == 0:
|
||||
logger.setLevel(logging.WARNING)
|
||||
elif args.v == 1:
|
||||
logger.setLevel(logging.INFO)
|
||||
elif args.v >= 2:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.debug(f'Debug logging')
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
coro = find_corrupt_videos()
|
||||
coro = find_corrupt_videos(channel_id=args.channel_id)
|
||||
loop.run_until_complete(coro)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ apt-get install -y \
|
|||
gnupg-agent \
|
||||
hostapd \
|
||||
htop \
|
||||
iotop \
|
||||
kiwix \
|
||||
kiwix-tools \
|
||||
libpq-dev \
|
||||
|
|
@ -34,6 +35,7 @@ apt-get install -y \
|
|||
python3-pip \
|
||||
python3-venv \
|
||||
software-properties-common \
|
||||
tmux \
|
||||
vim \
|
||||
zim-tools
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ apt-get install -y \
|
|||
gnupg-agent \
|
||||
hostapd \
|
||||
htop \
|
||||
iotop \
|
||||
kiwix \
|
||||
kiwix-tools \
|
||||
libpq-dev \
|
||||
|
|
@ -34,6 +35,7 @@ apt-get install -y \
|
|||
python3-pip \
|
||||
python3-venv \
|
||||
software-properties-common \
|
||||
tmux \
|
||||
vim \
|
||||
zim-tools
|
||||
|
||||
|
|
|
|||
|
|
@ -886,9 +886,22 @@ class DownloadManager:
|
|||
curs.execute(stmt)
|
||||
once_downloads = list(map(dict, curs.fetchall()))
|
||||
|
||||
stmt = '''
|
||||
SELECT
|
||||
COUNT(id) FILTER (
|
||||
WHERE frequency IS NULL
|
||||
AND (status = 'pending' OR status = 'new')
|
||||
)
|
||||
FROM
|
||||
download
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
pending_once_downloads = curs.fetchone()[0]
|
||||
|
||||
data = dict(
|
||||
recurring_downloads=recurring_downloads,
|
||||
once_downloads=once_downloads,
|
||||
pending_once_downloads=pending_once_downloads,
|
||||
)
|
||||
return data
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue