From 36083200e6f07c9eef0a96019ac528240239a4be Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 27 Jul 2023 15:53:47 -0600 Subject: [PATCH] Creating Video.ffprobe_json column which stores ffprobe information about the video file. Creating find_corrupt_videos.py script which automatically re-downloads videos which do not have both video and audio streams. --- .../8bdb48a486b9_video_ffprobe_json.py | 35 +++++ app/src/Events.js | 4 +- app/src/api.js | 26 ++-- app/src/components/Archive.js | 28 ++-- app/src/components/Common.js | 24 +++- app/src/components/DonatePage.js | 16 +++ app/src/components/Files.js | 49 +++---- app/src/components/VideoPlayer.js | 15 +- app/src/components/Videos.js | 44 +++--- app/src/hooks/customHooks.js | 5 + modules/archive/__init__.py | 30 +--- modules/archive/api.py | 5 +- modules/archive/conftest.py | 8 +- modules/archive/lib.py | 10 +- modules/archive/models.py | 2 +- modules/archive/schema.py | 8 +- modules/archive/test/test_api.py | 23 +++ modules/archive/test/test_lib.py | 48 +++---- modules/videos/__init__.py | 3 + modules/videos/channel/test/test_api.py | 9 +- modules/videos/common.py | 48 ++++++- modules/videos/downloader.py | 16 ++- modules/videos/lib.py | 6 +- modules/videos/models.py | 55 +++++++- modules/videos/test/test_common.py | 26 ++++ modules/videos/test/test_downloader.py | 2 + modules/videos/test/test_models.py | 15 ++ pi-gen/stage2/04-wrolpi/01-packages | 1 + scripts/find_corrupt_videos.py | 132 ++++++++++++++++++ scripts/install_debian_11.sh | 1 + scripts/install_raspberrypios.sh | 1 + scripts/start_kiwix_serve.sh | 13 +- wrolpi/common.py | 12 ++ wrolpi/conftest.py | 13 +- wrolpi/events.py | 4 +- wrolpi/files/api.py | 2 +- wrolpi/files/lib.py | 10 +- wrolpi/files/test/test_lib.py | 19 ++- wrolpi/test/test_events.py | 2 +- 39 files changed, 598 insertions(+), 172 deletions(-) create mode 100644 alembic/versions/8bdb48a486b9_video_ffprobe_json.py create mode 100644 modules/videos/test/test_models.py create mode 100755 scripts/find_corrupt_videos.py diff --git a/alembic/versions/8bdb48a486b9_video_ffprobe_json.py b/alembic/versions/8bdb48a486b9_video_ffprobe_json.py new file mode 100644 index 00000000..e9462107 --- /dev/null +++ b/alembic/versions/8bdb48a486b9_video_ffprobe_json.py @@ -0,0 +1,35 @@ +"""Video ffprobe json. + +Revision ID: 8bdb48a486b9 +Revises: bf704baa185d +Create Date: 2023-07-24 10:59:17.426100 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '8bdb48a486b9' +down_revision = 'bf704baa185d' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video ADD COLUMN ffprobe_json JSON') + # Re-index all videos so the ffprobe data is fetched. + session.execute("UPDATE file_group SET indexed=false WHERE mimetype like 'video/%'") + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS ffprobe_json') diff --git a/app/src/Events.js b/app/src/Events.js index d77d670e..db4bb765 100644 --- a/app/src/Events.js +++ b/app/src/Events.js @@ -16,6 +16,8 @@ function handleEvents(events) { const {event, message, subject, dt, url} = e; document.dispatchEvent(new CustomEvent(apiEventName, {detail: e})); + console.debug(`Got event: ${event}`); + if (event === 'directory_refresh') { eventToast( 'Refresh started', @@ -26,7 +28,7 @@ function handleEvents(events) { if (subject && newestEvents[subject] && newestEvents[subject] > dt) { console.debug(`Already handled newer event of "${subject}".`); return; - } else if (event === 'global_refresh_completed') { + } else if (event === 'refresh_completed') { eventToast('Refresh completed', 'All files have been refreshed.'); } diff --git a/app/src/api.js b/app/src/api.js index 93959a25..c390ed9e 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -539,23 +539,17 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta } } -export async function refreshFiles() { - return await apiPost(`${API_URI}/files/refresh`); -} - -export async function refreshDirectoryFiles(paths) { - let body = {paths: paths}; - try { - return await apiPost(`${API_URI}/files/refresh`, body); - } catch (e) { - console.error(e); - toast({ - type: 'error', - title: 'Unable to refresh directory', - description: 'Cannot refresh the files in the directory. See server logs.', - time: 5000, - }); +export async function refreshFiles(paths) { + let response; + if (paths && paths.constructor === Array) { + const body = {paths}; + console.debug(`Refreshing: ${paths}`); + response = await apiPost(`${API_URI}/files/refresh`, body); + } else { + console.debug(`Refreshing all files`); + response = await apiPost(`${API_URI}/files/refresh`); } + return response; } export async function makeDirectory(path) { diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index a314419e..7e8583e7 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -131,11 +131,11 @@ function ArchivePage() { historyList = ; } - const domain = data.domain ? data.domain.domain : null; + const domain = data.domain ? data.domain : null; let domainHeader; if (domain) { const domainUrl = `/archive?domain=${domain}`; - domainHeader =
+ domainHeader =
{domain} @@ -272,7 +272,7 @@ export function DomainsPage() { filteredDomains = domains.filter(i => re.test(i['domain'])); } - const row = ({domain, url_count}) => { + const domainRow = ({domain, url_count, size}) => { return @@ -280,12 +280,14 @@ export function DomainsPage() { {url_count} + {humanFileSize(size)} } const headers = [ {key: 'domain', text: 'Domain', sortBy: 'domain', width: 12}, {key: 'archives', text: 'Archives', sortBy: 'url_count', width: 2}, + {key: 'Size', text: 'Size', sortBy: 'size', width: 2}, ]; return <> @@ -298,7 +300,7 @@ export function DomainsPage() { row(i)} + rowFunc={(i, sortData) => domainRow(i)} rowKey='domain' tableHeaders={headers} /> @@ -362,11 +364,10 @@ function ArchivesPage() { } } - const onDelete = async (e) => { - e.preventDefault(); + const onDelete = async () => { setDeleteOpen(false); const archiveIds = archives.filter(i => selectedArchives.indexOf(i['primary_path']) >= 0) - .map(i => i['archive']['id']); + .map(i => i['data']['id']); await deleteArchives(archiveIds); await fetchArchives(); setSelectedArchives([]); @@ -383,18 +384,13 @@ function ArchivesPage() { } const selectElm =
- - setDeleteOpen(false)} - onConfirm={onDelete} - /> + confirmContent='Are you sure you want to delete these archives files? This cannot be undone.' + onClick={onDelete} + >Delete - setDeleteOpen(false)} - onConfirm={onDelete} - /> + confirmContent='Are you sure you want to delete these video files? This cannot be undone.' + onClick={onDelete} + >Delete