Creating Video.validated which is used by find_corrupt_videos.py to search.

This commit is contained in:
Roland 2023-08-08 13:42:55 -06:00
parent d0d315cd67
commit e8e5b5727f
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
17 changed files with 253 additions and 125 deletions

View file

@ -0,0 +1,31 @@
"""Video.valid.
Revision ID: f360a276cdcf
Revises: 8bdb48a486b9
Create Date: 2023-08-07 22:49:37.001794
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = 'f360a276cdcf'
down_revision = '8bdb48a486b9'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS validated')

View file

@ -33,7 +33,7 @@ function getEpubViewerURL(previewFile) {
}
}
function getIframeModal(previewFile) {
function getIframePreviewModal(previewFile) {
const url = getMediaPathURL(previewFile);
return <ModalContent>
<div className='full-height'>
@ -47,7 +47,7 @@ function getIframeModal(previewFile) {
</ModalContent>
}
function getImageModal(previewFile) {
function getImagePreviewModal(previewFile) {
const path = previewFile.primary_path ?? previewFile.path;
const url = getMediaPathURL(previewFile);
// Get the file name from the absolute path.
@ -64,7 +64,7 @@ function getImageModal(previewFile) {
</React.Fragment>
}
function getEpubModal(previewFile) {
function getEpubPreviewModal(previewFile) {
const downloadURL = getDownloadPathURL(previewFile);
const viewerUrl = `/epub/epub.html?url=${downloadURL}`;
return <ModalContent>
@ -77,11 +77,10 @@ function getEpubModal(previewFile) {
</ModalContent>
}
function getVideoModal(previewFile) {
function getVideoPreviewModal(previewFile) {
const url = getMediaPathURL(previewFile);
const path = previewFile.primary_path ?? previewFile.path;
const name = path.replace(/^.*[\\\/]/, '');
const type = previewFile.mimetype ? previewFile.mimetype : 'video/mp4';
return <React.Fragment>
<ModalHeader>
{name}
@ -94,13 +93,13 @@ function getVideoModal(previewFile) {
playsInline={true}
style={{maxWidth: '100%'}}
>
<source src={url} type={type}/>
<source src={url}/>
</video>
</ModalContent>
</React.Fragment>
}
function getAudioModal(previewFile) {
function getAudioPreviewModal(previewFile) {
const url = getMediaPathURL(previewFile);
const path = previewFile.primary_path ?? previewFile.path;
const name = path.replace(/^.*[\\\/]/, '');
@ -123,7 +122,7 @@ function getAudioModal(previewFile) {
</React.Fragment>
}
function getSTLModal(previewFile) {
function getSTLPreviewModal(previewFile) {
const url = getMediaPathURL(previewFile);
const path = previewFile.primary_path ?? previewFile.path;
const name = path.replace(/^.*[\\\/]/, '');
@ -139,7 +138,7 @@ function getSTLModal(previewFile) {
</React.Fragment>
}
function getGenericModal(previewFile) {
function getGenericPreviewModal(previewFile) {
// No special handler for this file type, just open it.
const path = previewFile.primary_path ?? previewFile.path;
const name = path.replace(/^.*[\\\/]/, '');
@ -275,27 +274,27 @@ export function FilePreviewProvider({children}) {
// Large text files should be downloaded.
window.open(downloadURL);
} else if (mimetype.startsWith('text/') || mimetype.startsWith('application/json')) {
setModalContent(getIframeModal(previewFile), url, downloadURL, path);
setModalContent(getIframePreviewModal(previewFile), url, downloadURL, path);
} else if (mimetype.startsWith('video/')) {
setModalContent(getVideoModal(previewFile), url, downloadURL, path);
setModalContent(getVideoPreviewModal(previewFile), url, downloadURL, path);
} else if (mimetype.startsWith('audio/')) {
setModalContent(getAudioModal(previewFile), url, downloadURL, path);
setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path);
} else if (mimetype.startsWith('application/epub')) {
const viewerURL = getEpubViewerURL(previewFile);
setModalContent(getEpubModal(previewFile), viewerURL, downloadURL, path);
setModalContent(getEpubPreviewModal(previewFile), viewerURL, downloadURL, path);
} else if (mimetype.startsWith('application/pdf')) {
setModalContent(getIframeModal(previewFile), url, downloadURL, path);
setModalContent(getIframePreviewModal(previewFile), url, downloadURL, path);
} else if (mimetype.startsWith('image/')) {
setModalContent(getImageModal(previewFile), url, null, path);
setModalContent(getImagePreviewModal(previewFile), url, null, path);
} else if (mimetype.startsWith('model/stl')) {
setModalContent(getSTLModal(previewFile), null, downloadURL, path);
setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path);
} else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.mp3')) {
setModalContent(getAudioModal(previewFile), url, downloadURL, path);
setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path);
} else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.stl')) {
setModalContent(getSTLModal(previewFile), null, downloadURL, path);
setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path);
} else {
// No special handler for this file type, just open it.
setModalContent(getGenericModal(previewFile), url, downloadURL, path);
setModalContent(getGenericPreviewModal(previewFile), url, downloadURL, path);
}
}
}, [previewFile]);

View file

@ -193,11 +193,6 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
}
let videoSource = <source src={videoUrl}/>;
if (video.codec_names && video.codec_names.indexOf('mp4') >= 0) {
videoSource = <source src={videoUrl} type="video/mp4"/>
} else if (video.codec_names && video.codec_names.indexOf('vp9') >= 0) {
videoSource = <source src={videoUrl} type='video/mp4;codecs="vp9, vorbis"'/>
}
return <>
<Container style={{margin: '1em'}}>

View file

@ -77,30 +77,14 @@ class DownloadRow extends React.Component {
super(props);
this.state = {
errorModalOpen: false,
deleteOpen: false,
}
}
openDelete = (e) => {
if (e) {
e.preventDefault();
}
this.setState({deleteOpen: true});
}
closeDelete = (e) => {
if (e) {
e.preventDefault();
}
this.setState({deleteOpen: false});
}
handleDelete = async () => {
const {id} = this.props;
try {
await deleteDownload(id);
} finally {
this.closeDelete();
if (this.props.fetchDownloads) {
this.props.fetchDownloads();
}
@ -109,7 +93,7 @@ class DownloadRow extends React.Component {
render() {
let {url, frequency, last_successful_download, status, location, next_download, error} = this.props;
const {errorModalOpen, deleteOpen} = this.state;
const {errorModalOpen} = this.state;
const link = location ?
(text) => <Link to={location}>{text}</Link> :
@ -171,9 +155,6 @@ class DownloadRow extends React.Component {
class StoppableRow extends React.Component {
constructor(props) {
super(props);
this.state = {
errorModalOpen: false,
};
}
handleDelete = async () => {
@ -194,7 +175,6 @@ class StoppableRow extends React.Component {
render() {
let {url, last_successful_download, status, location, error} = this.props;
let {errorModalOpen} = this.state;
// Open downloads (/download), or external links in an anchor.
const link = location && !location.startsWith('/download') ?
@ -326,13 +306,15 @@ export function RecurringDownloadsTable({downloads, fetchDownloads}) {
export function DownloadsPage() {
useTitle('Downloads');
const {onceDownloads, recurringDownloads, fetchDownloads} = useDownloads();
const {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads} = useDownloads();
const pendingOnceDownloadsSpan = pendingOnceDownloads > 0 ? <span>({pendingOnceDownloads})</span> : null;
return <>
<WROLModeMessage content='Downloads are disabled because WROL Mode is enabled.'/>
<DisableDownloadsToggle/>
<Header as='h1'>Downloads</Header>
<Header as='h1'>Downloads {pendingOnceDownloadsSpan}</Header>
<OnceDownloadsTable downloads={onceDownloads} fetchDownloads={fetchDownloads}/>
<Header as='h1'>Recurring Downloads</Header>

View file

@ -551,12 +551,14 @@ export const useHotspot = () => {
export const useDownloads = () => {
const [onceDownloads, setOnceDownloads] = useState(null);
const [recurringDownloads, setRecurringDownloads] = useState(null);
const [pendingOnceDownloads, setPendingOnceDownloads] = useState(null);
const fetchDownloads = async () => {
try {
const data = await getDownloads();
setOnceDownloads(data['once_downloads']);
setRecurringDownloads(data['recurring_downloads']);
setPendingOnceDownloads(data['pending_once_downloads']);
} catch (e) {
console.error(e);
}
@ -564,7 +566,7 @@ export const useDownloads = () => {
useRecurringTimeout(fetchDownloads, 3 * 1000);
return {onceDownloads, recurringDownloads, fetchDownloads}
return {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads}
}
export const useThrottle = () => {

View file

@ -44,7 +44,8 @@ async def video_modeler():
except Exception as e:
if PYTEST:
raise
logger.error(f'Unable to model Video {video_id=}', exc_info=e)
i = video.file_group.primary_path if video.file_group else video_id
logger.error(f'Unable to model Video {i=}', exc_info=e)
file_group.indexed = True

View file

@ -18,6 +18,7 @@ from sqlalchemy.orm import Session
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import FFPROBE_BIN
from wrolpi.common import logger, get_media_directory
from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
from .errors import ChannelNameConflict, ChannelURLConflict, ChannelDirectoryConflict, ChannelSourceIdConflict
@ -118,24 +119,45 @@ def get_duration_from_ffmpeg(stderr: str) -> int:
return duration
def generate_video_poster(video_path: Path) -> Tuple[Path, Optional[int]]:
"""Create a poster (aka thumbnail) next to the provided video_path. Also returns the video duration."""
poster_path = video_path.with_suffix('.jpg')
cmd = (FFMPEG_BIN, '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000',
str(poster_path))
if poster_path.exists():
# Poster already exists.
return poster_path, None
def ffmpeg_poster(video_path: Path, poster_path: Path, seconds: int):
"""Use ffmpeg to create a poster file from a video file at a particular time of the video.
Returns stderr so data about the command can be extracted."""
timestamp = seconds_to_timestamp(seconds)
cmd = (FFMPEG_BIN, '-ss', timestamp, '-n', '-i', video_path, '-f', 'mjpeg', '-vframes', '1', poster_path)
try:
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stderr = proc.stderr.decode()
logger.info(f'Generated poster at {poster_path}')
except subprocess.CalledProcessError as e:
logger.warning(f'FFMPEG poster generation failed with stdout: {e.stdout.decode()}')
logger.warning(f'FFMPEG poster generation failed with stdout: {e.stderr.decode()}')
logger.error(f'Failed to create video poster', exc_info=e)
raise
if not poster_path.exists():
raise Exception(f'Failed to find generated poster! {poster_path}')
return stderr
def ffmpeg_video_complete(video_path: Path, seconds: int = None) -> bool:
"""Checks if video file is complete by taking screenshot from the end of the video."""
seconds = seconds or get_video_duration(video_path) - 5
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as fh:
path = Path(fh.name)
path.unlink()
try:
ffmpeg_poster(video_path, path, seconds)
return True
except Exception:
return False
finally:
path.unlink()
def generate_video_poster(video_path: Path, seconds: int = 5) -> Tuple[Path, Optional[int]]:
"""Create a poster (aka thumbnail) next to the provided video_path. Also returns the video duration."""
poster_path = video_path.with_suffix('.jpg')
if poster_path.exists():
# Poster already exists.
return poster_path, None
stderr = ffmpeg_poster(video_path, poster_path, seconds)
duration = get_duration_from_ffmpeg(stderr)
return poster_path, duration

View file

@ -22,7 +22,7 @@ from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.lib import glob_shared_stem
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import get_no_channel_directory, get_videos_directory, update_view_counts
from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete
from .errors import UnknownChannel, ChannelURLEmpty
from .lib import get_downloader_config
from .models import Video, Channel
@ -168,6 +168,9 @@ class VideoDownloader(Downloader, ABC):
"""Downloads a single video.
Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`.
Videos are validated in a few ways. A video file is valid when it has a complete video stream and an audio stream.
A complete video stream is validated by taking a screenshot at the end of the video.
"""
name = 'video'
pretty_name = 'Videos'
@ -300,6 +303,12 @@ class VideoDownloader(Downloader, ABC):
error=error,
)
if not ffmpeg_video_complete(video_path):
return DownloadResult(
success=False,
error='Video was incomplete',
)
with get_db_session(commit=True) as session:
# Find any files downloaded with the video (poster, caption, etc.).
video_paths = glob_shared_stem(video_path)

View file

@ -119,8 +119,11 @@ def validate_video(video: Video, channel_generate_poster: bool):
if video_path and not video.duration:
# Duration was not retrieved during poster generation.
if video.ffprobe_json and (video_streams := video.get_streams_by_codec_type('video')):
video.duration = float(video_streams[0]['duration'])
if video.ffprobe_json:
if duration := video.ffprobe_json['format'].get('duration'):
video.duration = float(duration)
elif video_streams := video.get_streams_by_codec_type('video'):
video.duration = float(video_streams[0]['duration'])
else:
# Slowest method.
video.duration = get_video_duration(video_path)

View file

@ -33,6 +33,7 @@ class Video(ModelHelper, Base):
source_id = Column(String)
upload_date: datetime.datetime = Column(TZDateTime)
url = Column(String)
validated = Column(Boolean, default=False)
view_count = Column(Integer)
viewed = Column(TZDateTime)
ffprobe_json = deferred(Column(JSON))
@ -86,12 +87,13 @@ class Video(ModelHelper, Base):
)
return d
def delete(self):
def delete(self, add_to_skip_list: bool = True):
"""Remove all files and File records related to this video. Delete this Video record.
Add it to it's Channel's skip list."""
self.file_group.delete()
self.add_to_skip_list()
if add_to_skip_list:
self.add_to_skip_list()
session = Session.object_session(self)
session.delete(self)
@ -340,6 +342,10 @@ class Video(ModelHelper, Base):
streams = [i for i in self.ffprobe_json['streams'] if i['codec_type'] == codec_type]
return streams
def detect_is_complete(self):
from modules.videos.common import ffmpeg_video_complete
return ffmpeg_video_complete(self.video_path)
class Channel(ModelHelper, Base):
__tablename__ = 'channel'

View file

@ -123,9 +123,10 @@ def test_generate_video_poster(video_file):
A poster can be generated from a video file.
"""
poster_path = video_file.with_suffix('.jpg')
generate_video_poster(video_file)
_, duration = generate_video_poster(video_file)
assert poster_path.is_file(), f'{poster_path} was not created!'
assert poster_path.stat().st_size > 0
assert duration == 5
def test_import_channel_downloads(test_session, channel_factory, test_channels_config):

View file

@ -319,7 +319,7 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
@pytest.mark.asyncio
async def test_video_download(test_session, test_directory, simple_channel, video_download_manager,
mock_video_process_runner, image_file):
"""A video download is performed, files are associated."""
"""A video download is performed, files are grouped."""
simple_channel.source_id = example_video_json['channel_id']
simple_channel.directory = test_directory / 'videos/channel name'
simple_channel.directory.mkdir(parents=True)

View file

@ -20,6 +20,7 @@ gdal-bin
git
gnupg-agent
hostapd
iotop
kiwix
kiwix-tools
libagg-dev
@ -74,6 +75,7 @@ python3-venv
renderd
software-properties-common
tar
tmux
ttf-unifont
unzip
vim

View file

@ -7,37 +7,28 @@ will be prompted to delete it.
import argparse
import asyncio
import logging
import pathlib
import subprocess
import sys
import tempfile
from http import HTTPStatus
from typing import List
import aiohttp
from sqlalchemy.orm import Session
from yt_dlp.utils import YoutubeDLError
from modules.videos.downloader import VideoDownloader, extract_info
from modules.videos.models import Video
from modules.videos.downloader import extract_info, VideoDownloader
from modules.videos.models import Video, Channel
from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import download_manager
logger = logging.getLogger()
def get_total_corrupted_videos() -> int:
with get_db_curs() as curs:
stmt = '''
WITH all_videos AS (
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
from video v
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
group by v.id)
SELECT COUNT(all_videos.id)
FROM all_videos
WHERE NOT all_videos.codec_types @> '{audio,video}'::TEXT[]
'''
curs.execute(stmt)
return int(curs.fetchone()[0])
def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
params = dict(limit=limit, offset=offset)
def get_incomplete_videos(limit: int, offset: int, channel_id: int) -> List[dict]:
"""Returns Video records that do not have both video and audio streams."""
params = dict(limit=limit, offset=offset, channel_id=channel_id)
with get_db_curs() as curs:
stmt = '''
WITH all_videos AS (
@ -46,6 +37,7 @@ def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types
from video v
cross join lateral json_array_elements(ffprobe_json -> 'streams') s
where channel_id = %(channel_id)s
group by v.id)
SELECT all_videos.id AS video_id, fg.primary_path, all_videos.codec_types
FROM all_videos
@ -62,16 +54,26 @@ def get_corrupt_videos(limit: int, offset: int) -> List[dict]:
return [dict(i) for i in curs.fetchall()]
def iterate_corrupt_videos() -> List[dict]:
def iterate_incomplete_videos(channel_id: int = None) -> List[dict]:
limit = 20
offset = 0
while True:
videos = get_corrupt_videos(limit, offset)
offset += limit
if len(videos) < limit:
# Ran out of corrupt videos.
break
yield videos
with get_db_session() as session:
channel_ids = [channel_id, ] if channel_id else [int(i[0]) for i in session.query(Channel.id)]
found = False
for channel_id in channel_ids:
channel_id: int
while True:
videos = get_incomplete_videos(limit, offset, channel_id)
offset += limit
if videos:
yield videos
if len(videos) < limit:
# Ran out of corrupt videos.
if found:
logger.debug(f'Ran out of corrupt videos for {channel_id=}')
else:
logger.debug(f'No corrupt videos for {channel_id=}')
break
def confirm(msg: str) -> bool:
@ -86,47 +88,103 @@ def confirm(msg: str) -> bool:
return False
async def find_corrupt_videos():
total_corrupted = get_total_corrupted_videos()
count = 0
def can_be_downloaded(url: str) -> bool:
try:
extract_info(url)
return True
except YoutubeDLError:
return False
async def create_download(url: str, downloader: str):
json_ = {'urls': url, 'downloader': downloader}
async with aiohttp.ClientSession() as session:
async with session.post('http://127.0.0.1:8080/api/download', json=json_) as response:
if response.status == HTTPStatus.NO_CONTENT:
logger.info(f'Created download for {url}')
else:
raise Exception(f'Failed to create download for {url}')
async def download_or_ask_delete(video: Video, session: Session, delete_message: str):
if video.url and can_be_downloaded(video.url):
await create_download(video.url, VideoDownloader.name)
# Always re-download when possible.
do_delete = True
else:
logger.error(f'Video cannot be downloaded: {video.url or video}')
do_delete = confirm(delete_message)
if do_delete:
logger.debug(f'Deleting {video}')
video.delete(add_to_skip_list=False)
session.commit()
async def find_corrupt_videos(channel_id: int = None):
with get_db_session() as session:
for chunk in iterate_corrupt_videos():
for chunk in iterate_incomplete_videos(channel_id=channel_id):
video_ids = [i['video_id'] for i in chunk]
videos = session.query(Video).filter(Video.id.in_(video_ids)).all()
for video in videos:
print()
print(f'Video (remaining: {total_corrupted - count}): {video}')
count += 1
streams = (await video.get_ffprobe_json())['streams']
codec_names = [i['codec_name'] for i in streams]
print(f'codecs: {codec_names}')
path = video.video_path
await download_or_ask_delete(video, session, f'Missing audio. {path} Delete? (y/N)')
url = video.url
if url:
try:
extract_info(url)
print(f'Re-downloading')
download_manager.create_download(url, VideoDownloader.name, session=session)
video.delete()
with get_db_session() as session:
videos = session.query(Video).filter(Video.validated == False) # noqa
if channel_id:
videos = videos.filter(Video.channel_id == channel_id)
for video in videos:
if not video.duration:
logger.warning(f'No duration for {video}')
continue
if video.duration > 5:
timestamp = seconds_to_timestamp(video.duration - 5)
else:
timestamp = seconds_to_timestamp(video.duration - 1)
# Create a screenshot from the end of the video. If this fails, the video is corrupt.
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as fh:
fh = pathlib.Path(fh.name)
fh.unlink()
try:
cmd = (
'ffmpeg', '-n',
'-ss', timestamp,
'-i', str(video.video_path),
'-f', 'mjpeg',
'-vframes', '1',
'-q:v', '2',
str(fh),
)
proc = subprocess.run(cmd, check=True, capture_output=True, timeout=10)
if fh.is_file() and fh.stat().st_size:
logger.debug(f'{video.video_path.name} is valid')
video.validated = True
session.commit()
fh.unlink()
continue
except YoutubeDLError:
logger.error(f'Video cannot be downloaded: {url}')
if confirm(f'Delete? (y/N)'):
video.delete()
session.commit()
continue
except subprocess.CalledProcessError as e:
logger.error(f'Failed to generate poster for {video.video_path}', exc_info=e)
if confirm('Delete? (y/N)'):
video.delete()
session.commit()
path = video.video_path
await download_or_ask_delete(video, session, f'Corrupt video. {path} Delete? (y/N)')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-v', action='count')
parser.add_argument('-c', '--channel-id', type=int,
help='The id of the channel to search')
args = parser.parse_args()
if args.v == 0:
logger.setLevel(logging.WARNING)
elif args.v == 1:
logger.setLevel(logging.INFO)
elif args.v >= 2:
logger.setLevel(logging.DEBUG)
logger.debug(f'Debug logging')
loop = asyncio.get_event_loop()
coro = find_corrupt_videos()
coro = find_corrupt_videos(channel_id=args.channel_id)
loop.run_until_complete(coro)

View file

@ -20,6 +20,7 @@ apt-get install -y \
gnupg-agent \
hostapd \
htop \
iotop \
kiwix \
kiwix-tools \
libpq-dev \
@ -34,6 +35,7 @@ apt-get install -y \
python3-pip \
python3-venv \
software-properties-common \
tmux \
vim \
zim-tools

View file

@ -20,6 +20,7 @@ apt-get install -y \
gnupg-agent \
hostapd \
htop \
iotop \
kiwix \
kiwix-tools \
libpq-dev \
@ -34,6 +35,7 @@ apt-get install -y \
python3-pip \
python3-venv \
software-properties-common \
tmux \
vim \
zim-tools

View file

@ -886,9 +886,22 @@ class DownloadManager:
curs.execute(stmt)
once_downloads = list(map(dict, curs.fetchall()))
stmt = '''
SELECT
COUNT(id) FILTER (
WHERE frequency IS NULL
AND (status = 'pending' OR status = 'new')
)
FROM
download
'''
curs.execute(stmt)
pending_once_downloads = curs.fetchone()[0]
data = dict(
recurring_downloads=recurring_downloads,
once_downloads=once_downloads,
pending_once_downloads=pending_once_downloads,
)
return data