mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Creating Video.ffprobe_json column which stores ffprobe information about the video file. Creating find_corrupt_videos.py script which automatically re-downloads videos which do not have both video and audio streams.
This commit is contained in:
parent
097c3143e1
commit
36083200e6
39 changed files with 598 additions and 172 deletions
132
scripts/find_corrupt_videos.py
Executable file
132
scripts/find_corrupt_videos.py
Executable file
|
|
@ -0,0 +1,132 @@
|
|||
#! /usr/bin/env python3
|
||||
"""
|
||||
Searches the DB for Video records which do not have both video and audio streams. If a Video still exists at its URL,
|
||||
it will be automatically re-downloaded. If a Video cannot be re-downloaded (it has been deleted from its host) you
|
||||
will be prompted to delete it.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
from yt_dlp.utils import YoutubeDLError
|
||||
|
||||
from modules.videos.downloader import VideoDownloader, extract_info
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import download_manager
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
def get_total_corrupted_videos() -> int:
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
WITH all_videos AS (
|
||||
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
|
||||
from video v
|
||||
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
|
||||
group by v.id)
|
||||
SELECT COUNT(all_videos.id)
|
||||
FROM all_videos
|
||||
WHERE NOT all_videos.codec_types @> '{audio,video}'::TEXT[]
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
return int(curs.fetchone()[0])
|
||||
|
||||
|
||||
def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
|
||||
params = dict(limit=limit, offset=offset)
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
WITH all_videos AS (
|
||||
-- Extract 'codec_type' from all 'stream' objects of each video.
|
||||
-- {'streams': [{'codec_type': 'video'}, {'codec_type': 'audio'}]} -> {video,audio}
|
||||
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
|
||||
from video v
|
||||
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
|
||||
group by v.id)
|
||||
SELECT all_videos.id AS video_id, fg.primary_path, all_videos.codec_types
|
||||
FROM all_videos
|
||||
LEFT JOIN video v ON v.id = all_videos.id
|
||||
LEFT JOIN public.file_group fg on fg.id = v.file_group_id
|
||||
-- Filter out videos that have video AND audio streams.
|
||||
WHERE NOT all_videos.codec_types @> '{video,audio}'::TEXT[]
|
||||
AND fg.primary_path like '/media/wrolpi/videos/%%'
|
||||
ORDER BY all_videos.id
|
||||
LIMIT %(limit)s
|
||||
OFFSET %(offset)s
|
||||
'''
|
||||
curs.execute(stmt, params)
|
||||
return [dict(i) for i in curs.fetchall()]
|
||||
|
||||
|
||||
def iterate_corrupt_videos() -> List[dict]:
|
||||
limit = 20
|
||||
offset = 0
|
||||
while True:
|
||||
videos = get_corrupt_videos(limit, offset)
|
||||
offset += limit
|
||||
if len(videos) < limit:
|
||||
# Ran out of corrupt videos.
|
||||
break
|
||||
yield videos
|
||||
|
||||
|
||||
def confirm(msg: str) -> bool:
|
||||
while True:
|
||||
input_ = input(msg)
|
||||
if input_ == 'y':
|
||||
return True
|
||||
if input_ == 'q':
|
||||
print('Quitting...')
|
||||
sys.exit(0)
|
||||
if not input_ or input_ == 'n':
|
||||
return False
|
||||
|
||||
|
||||
async def find_corrupt_videos():
|
||||
total_corrupted = get_total_corrupted_videos()
|
||||
count = 0
|
||||
|
||||
with get_db_session() as session:
|
||||
for chunk in iterate_corrupt_videos():
|
||||
video_ids = [i['video_id'] for i in chunk]
|
||||
videos = session.query(Video).filter(Video.id.in_(video_ids)).all()
|
||||
for video in videos:
|
||||
print()
|
||||
print(f'Video (remaining: {total_corrupted - count}): {video}')
|
||||
count += 1
|
||||
streams = (await video.get_ffprobe_json())['streams']
|
||||
codec_names = [i['codec_name'] for i in streams]
|
||||
print(f'codecs: {codec_names}')
|
||||
|
||||
url = video.url
|
||||
if url:
|
||||
try:
|
||||
extract_info(url)
|
||||
print(f'Re-downloading')
|
||||
download_manager.create_download(url, VideoDownloader.name, session=session)
|
||||
video.delete()
|
||||
session.commit()
|
||||
continue
|
||||
except YoutubeDLError:
|
||||
logger.error(f'Video cannot be downloaded: {url}')
|
||||
if confirm(f'Delete? (y/N)'):
|
||||
video.delete()
|
||||
session.commit()
|
||||
continue
|
||||
|
||||
if confirm('Delete? (y/N)'):
|
||||
video.delete()
|
||||
session.commit()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
coro = find_corrupt_videos()
|
||||
loop.run_until_complete(coro)
|
||||
|
|
@ -23,6 +23,7 @@ apt-get install -y \
|
|||
kiwix \
|
||||
kiwix-tools \
|
||||
libpq-dev \
|
||||
netcat \
|
||||
network-manager \
|
||||
nginx-doc \
|
||||
nginx-full \
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ apt-get install -y \
|
|||
kiwix \
|
||||
kiwix-tools \
|
||||
libpq-dev \
|
||||
netcat \
|
||||
network-manager \
|
||||
nginx-doc \
|
||||
nginx-full \
|
||||
|
|
|
|||
|
|
@ -1,16 +1,25 @@
|
|||
#! /usr/bin/env bash
|
||||
# This file imports all *.zim files in the media directory, then starts kiwix-serve.
|
||||
# This file imports all *.zim files in the Zim media directory, then starts kiwix-serve.
|
||||
|
||||
[ ! -d /media/wrolpi ] && echo "Cannot start kiwix without media directory mounted" && exit 1
|
||||
|
||||
LIBRARY=/media/wrolpi/zims/library.xml
|
||||
|
||||
# Wait for the zims directory to be mounted.
|
||||
COUNT=0
|
||||
while [ ${COUNT} -lt 30 ]; do
|
||||
if [ -d /media/wrolpi/zims ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Create the zim directory, if necessary.
|
||||
[ ! -d /media/wrolpi/zims ] && mkdir /media/wrolpi/zims
|
||||
# We will replace the library file.
|
||||
rm -f ${LIBRARY}
|
||||
# Find all Zim files and import them.
|
||||
find /media/wrolpi -iname '*.zim' -exec kiwix-manage ${LIBRARY} add {} \;
|
||||
find /media/wrolpi/zims -iname '*.zim' -exec kiwix-manage ${LIBRARY} add {} \;
|
||||
|
||||
[ ! -f ${LIBRARY} ] && echo "Could not find any Zim files to import" && exit 2
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue