mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
A download records the sha256 of its files in the wrolpi section (#904)
* A download records the sha256 of its files in the wrolpi section A video download writes `wrolpi.sha256` into its .info.json; an archive download writes it into its .readability.json. Sums are keyed by suffix, so a rename or Reorganize keeps them valid, and the metadata JSON holding them is not hashed. Large files are hashed in execute_download (in a thread, without the DB session). Finalize reuses those sums while a file's mtime is unchanged and hashes only files finalize created or changed (a converted poster, the ffprobe json). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * A video download hashes only for an info json, and finalize never suspends Finalize holds the write session; awaiting the hash let the loop run a switch whose write waited on that session, stalling the loop for the busy timeout. Finalize now hashes inline: the large files were hashed in execute_download. A download that writes no info json has nowhere to keep the sums, so it no longer reads its files to hash them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a40beac0f6
commit
32405a9132
6 changed files with 245 additions and 5 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
|
|
@ -17,6 +18,7 @@ from wrolpi.db import get_db_session
|
|||
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.modeler import run_modeler_loop, skip_clause, SkipModeler
|
||||
from wrolpi.files.lib import sha256_files, sha256_by_suffix, FileSums
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED, DOWNLOAD_USER_AGENT, PROJECT_DIR
|
||||
from . import lib
|
||||
|
|
@ -48,6 +50,7 @@ class PreparedArchive:
|
|||
class ExecutedArchive:
|
||||
"""Output of ArchiveDownloader.execute_download: files are on disk and ready to register."""
|
||||
written: WrittenArchiveFiles
|
||||
sha256: FileSums = None # sums of the written files, hashed without the DB session
|
||||
|
||||
|
||||
class ArchiveDownloader(Downloader, ABC):
|
||||
|
|
@ -98,11 +101,13 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
|
||||
written = write_archive_files(artifacts.url, artifacts.singlefile, artifacts.readability,
|
||||
artifacts.screenshot, destination=prepared.destination)
|
||||
return ExecutedArchive(written=written)
|
||||
sha256 = await asyncio.to_thread(sha256_files, written.paths)
|
||||
return ExecutedArchive(written=written, sha256=sha256)
|
||||
|
||||
def finalize_download(self, session: Session, download: Download,
|
||||
executed: ExecutedArchive) -> DownloadResult:
|
||||
"""Register the Archive record, apply tags, persist parent_download_url. Caller commits."""
|
||||
"""Register the Archive record, apply tags, persist parent_download_url and the sha256 of its files.
|
||||
Caller commits."""
|
||||
archive = register_archive(session, executed.written)
|
||||
|
||||
if tag_names := download.tag_names:
|
||||
|
|
@ -116,6 +121,12 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
except ValueError as e:
|
||||
logger.warning(f'Failed to update wrolpi json for {archive}: {e}')
|
||||
|
||||
try:
|
||||
sums = sha256_files(archive.file_group.my_paths(), executed.sha256)
|
||||
archive.file_group.update_wrolpi_json({'sha256': sha256_by_suffix(sums)})
|
||||
except (OSError, ValueError) as e:
|
||||
logger.warning(f'Failed to record sha256 of {archive}', exc_info=e)
|
||||
|
||||
logger.info(f'Successfully downloaded Archive {download.url} {archive}')
|
||||
return DownloadResult(success=True, location=archive.location)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@ isolation. None of them use async_client or test_download_manager — that's th
|
|||
whole point of the refactor.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
import modules.archive
|
||||
from modules.archive import (
|
||||
ExecutedArchive,
|
||||
PreparedArchive,
|
||||
|
|
@ -266,3 +269,32 @@ async def test_do_singlefile_configured_browser(test_session, test_directory, mo
|
|||
|
||||
capture, = calls
|
||||
assert capture['env']['WROLPI_BROWSER'] == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_records_sha256(async_client, test_session, test_directory, monkeypatch):
|
||||
"""An archive download records the sha256 of each file in its group in the readability json's wrolpi section."""
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
|
||||
async def mock_request_archive(url, compress=False):
|
||||
return singlefile, readability, screenshot
|
||||
|
||||
monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)
|
||||
|
||||
download = Download(url='https://example.com/x', downloader='archive',
|
||||
settings={'parent_download_url': 'https://example.com/feed'})
|
||||
prepared = archive_downloader.prepare_download(test_session, download)
|
||||
executed = await archive_downloader.execute_download(prepared, make_test_ctx())
|
||||
result = archive_downloader.finalize_download(test_session, download, executed)
|
||||
test_session.commit()
|
||||
assert result.success is True
|
||||
|
||||
archive = test_session.query(Archive).one()
|
||||
readability_json_path = archive.file_group.readability_json_path
|
||||
wrolpi = json.loads(readability_json_path.read_text())['wrolpi']
|
||||
assert wrolpi['parent_download_url'] == 'https://example.com/feed'
|
||||
stem = archive.singlefile_path.name[:-len('.html')]
|
||||
expected = {path.name[len(stem):]: hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in archive.file_group.my_paths() if path != readability_json_path}
|
||||
assert set(expected) == {'.html', '.readability.html', '.readability.txt', '.png'}
|
||||
assert wrolpi['sha256'] == expected
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#! /usr/bin/env python3
|
||||
import asyncio
|
||||
import copy
|
||||
import dataclasses
|
||||
import json
|
||||
|
|
@ -30,7 +31,8 @@ from wrolpi.db import get_db_session
|
|||
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \
|
||||
clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config
|
||||
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError
|
||||
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
|
||||
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix, sha256_files, sha256_by_suffix, \
|
||||
FileSums
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
|
||||
|
|
@ -923,6 +925,7 @@ class ExecutedVideo:
|
|||
settings: dict
|
||||
location: Optional[str] # for error reporting
|
||||
wrolpi: dict = field(default_factory=dict) # existing video's wrolpi section, to carry over
|
||||
sha256: FileSums = field(default_factory=dict) # sums of video_paths, hashed without the DB session
|
||||
|
||||
|
||||
class VideoDownloader(Downloader, ABC):
|
||||
|
|
@ -1209,6 +1212,12 @@ class VideoDownloader(Downloader, ABC):
|
|||
except Exception:
|
||||
logger.warning(f'Failed to center YouTube captions in {path}', exc_info=True)
|
||||
|
||||
# Hash the (large) files here; finalize holds the DB session. The sums are kept in the info json, so
|
||||
# without one there is nothing to hash for.
|
||||
sha256 = dict()
|
||||
if effective['writeinfojson'] or any(i.name.endswith('.info.json') for i in video_paths):
|
||||
sha256 = await asyncio.to_thread(sha256_files, video_paths)
|
||||
|
||||
return ExecutedVideo(
|
||||
url=url,
|
||||
video_path=video_path,
|
||||
|
|
@ -1221,6 +1230,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
settings=settings,
|
||||
location=location,
|
||||
wrolpi=prepared.wrolpi,
|
||||
sha256=sha256,
|
||||
)
|
||||
|
||||
except BotBlockedDownloadError:
|
||||
|
|
@ -1300,6 +1310,16 @@ class VideoDownloader(Downloader, ABC):
|
|||
error='Download did not contain audio stream',
|
||||
location=location)
|
||||
|
||||
# Hashed once the files are final: validation may have converted the poster, or written the ffprobe json.
|
||||
# Not awaited: this session holds the write lock, so suspending would let a switch's write stall the loop.
|
||||
# Only files changed since execute_download are read, and those are small.
|
||||
try:
|
||||
if video.info_json_path:
|
||||
sums = sha256_files(video.file_group.my_paths(), executed.sha256)
|
||||
video.file_group.update_wrolpi_json({'sha256': sha256_by_suffix(sums)})
|
||||
except (OSError, ValueError) as e:
|
||||
logger.warning(f'Failed to record sha256 of {video}', exc_info=e)
|
||||
|
||||
logger.info(f'Successfully downloaded video {executed.url} {video}')
|
||||
|
||||
# Delete duplicate Videos that share the URL or source_id. Runs against the same
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from copy import copy
|
||||
|
|
@ -10,7 +11,8 @@ import pytest
|
|||
from modules.videos.downloader import VideoDownloader, \
|
||||
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
|
||||
prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \
|
||||
parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP
|
||||
parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP, \
|
||||
ExecutedVideo
|
||||
from modules.videos.lib import get_videos_downloader_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.conftest import test_directory, await_switches
|
||||
|
|
@ -1992,5 +1994,115 @@ async def test_redownload_keeps_wrolpi_section(test_session, test_directory, moc
|
|||
video: Video = test_session.query(Video).one()
|
||||
info_json = video.get_info_json()
|
||||
assert info_json['title'] == 'Orig v2'
|
||||
assert info_json['wrolpi'] == {'custom_title': 'Mine'}
|
||||
ffprobe_json_path = video_path.with_suffix('.ffprobe.json')
|
||||
assert info_json['wrolpi'] == {
|
||||
'custom_title': 'Mine',
|
||||
'sha256': {'.mp4': hashlib.sha256(video_path.read_bytes()).hexdigest(),
|
||||
'.ffprobe.json': hashlib.sha256(ffprobe_json_path.read_bytes()).hexdigest()},
|
||||
}
|
||||
assert video.file_group.title == 'Mine'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_records_sha256(test_session, test_directory, mock_video_extract_info, simple_channel,
|
||||
await_switches, video_download_manager, mock_video_process_runner,
|
||||
image_bytes_factory):
|
||||
"""A download records the sha256 of each file in its group, as the files are once the download finishes."""
|
||||
simple_channel.source_id = example_video_json['channel_id']
|
||||
simple_channel.directory = test_directory / 'videos/channel name'
|
||||
simple_channel.directory.mkdir(parents=True)
|
||||
simple_channel.generate_posters = True
|
||||
test_session.commit()
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
video_path = simple_channel.directory / 'a video.mp4'
|
||||
info_json_path = video_path.with_suffix('.info.json')
|
||||
|
||||
from wrolpi.cmd import CommandResult
|
||||
|
||||
async def yt_dlp_writes_files(*args, **kwargs):
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
||||
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', video_path.with_suffix('.en.vtt'))
|
||||
# A WebP poster, which is converted to a JPEG once downloaded.
|
||||
video_path.with_suffix('.webp').write_bytes(image_bytes_factory('.webp'))
|
||||
info_json_path.write_text(json.dumps({'title': 'Title', 'webpage_url': url, 'duration': 5}))
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
|
||||
mock_video_process_runner.side_effect = yt_dlp_writes_files
|
||||
with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
|
||||
mock_video_extract_info.return_value = example_video_json
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
video_download_manager.create_download(test_session, url, video_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
test_session.expire_all()
|
||||
video: Video = test_session.query(Video).one()
|
||||
sums = video.get_info_json()['wrolpi']['sha256']
|
||||
assert not video_path.with_suffix('.webp').exists()
|
||||
assert '.webp' not in sums, 'the converted poster is gone'
|
||||
assert '.info.json' not in sums, 'the info json holds the sums'
|
||||
expected = {path.name[len('a video'):]: hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in video.file_group.my_paths() if path != info_json_path and path.is_file()}
|
||||
assert {'.mp4', '.en.vtt', '.jpg'} <= set(expected)
|
||||
assert sums == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_never_suspends(test_session, test_directory):
|
||||
"""finalize_download holds the write session from start to end. Were it to suspend, the event loop could run a
|
||||
switch whose (blocking) write waits on that session, and the loop would stall until the database's busy timeout.
|
||||
So once the files are on disk, finalize completes without suspending."""
|
||||
directory = test_directory / 'videos/channel name'
|
||||
directory.mkdir(parents=True)
|
||||
video_path = directory / 'a video.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
||||
info_json_path = video_path.with_suffix('.info.json')
|
||||
url = 'https://example.com/the-video'
|
||||
info_json_path.write_text(json.dumps({'id': 'the-id', 'title': 'Title', 'webpage_url': url}))
|
||||
executed = ExecutedVideo(url=url, video_path=video_path, video_paths=[video_path, info_json_path],
|
||||
info_json=dict(), entry={'id': 'the-id'}, channel_id=None, audio_only=False,
|
||||
tag_names=[], settings=dict(), location=None)
|
||||
|
||||
coro = video_downloader.finalize_download(test_session, Download(url=url, downloader='video'), executed)
|
||||
try:
|
||||
coro.send(None)
|
||||
except StopIteration as e:
|
||||
result = e.value
|
||||
else:
|
||||
coro.close()
|
||||
pytest.fail('finalize_download suspended')
|
||||
|
||||
assert result.success is True, result
|
||||
sums = json.loads(info_json_path.read_text())['wrolpi']['sha256']
|
||||
assert sums['.mp4'] == hashlib.sha256(video_path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_without_info_json_skips_sha256(test_session, test_directory, mock_video_extract_info,
|
||||
simple_channel, await_switches, video_download_manager,
|
||||
mock_video_process_runner):
|
||||
"""Sums are kept in the info json; a download that writes none does not read its files to hash them."""
|
||||
simple_channel.source_id = example_video_json['channel_id']
|
||||
simple_channel.directory = test_directory / 'videos/channel name'
|
||||
simple_channel.directory.mkdir(parents=True)
|
||||
test_session.commit()
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
video_path = simple_channel.directory / 'a video.mp4'
|
||||
|
||||
from wrolpi.cmd import CommandResult
|
||||
|
||||
async def yt_dlp_writes_video(*args, **kwargs):
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
|
||||
mock_video_process_runner.side_effect = yt_dlp_writes_video
|
||||
with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.sha256_files') as mock_sha256_files:
|
||||
mock_video_extract_info.return_value = example_video_json
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
video_download_manager.create_download(test_session, url, video_downloader.name,
|
||||
settings={'writeinfojson': False})
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.info_json_path is None
|
||||
mock_sha256_files.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import dataclasses
|
|||
import datetime
|
||||
import functools
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
|
|
@ -692,6 +693,38 @@ def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...
|
|||
return results
|
||||
|
||||
|
||||
# The metadata JSON holds the sums, so it cannot hold its own.
|
||||
METADATA_JSON_SUFFIXES = ('.info.json', '.readability.json')
|
||||
|
||||
# A file's (modification time in nanoseconds, hex sha256), keyed by its path.
|
||||
FileSums = Dict[pathlib.Path, Tuple[int, str]]
|
||||
|
||||
|
||||
def sha256_files(paths: Iterable[pathlib.Path], known: FileSums = None) -> FileSums:
|
||||
"""Hash each file, skipping the metadata JSON and files that do not exist.
|
||||
|
||||
A `known` sum is reused while its file's modification time is unchanged. This reads whole files; call it in a
|
||||
thread."""
|
||||
known = known or dict()
|
||||
sums = dict()
|
||||
for path in paths:
|
||||
if path.name.endswith(METADATA_JSON_SUFFIXES) or not path.is_file():
|
||||
continue
|
||||
mtime_ns = path.stat().st_mtime_ns
|
||||
if (known_sum := known.get(path)) and known_sum[0] == mtime_ns:
|
||||
sums[path] = known_sum
|
||||
continue
|
||||
with path.open('rb') as fh:
|
||||
sums[path] = (mtime_ns, hashlib.file_digest(fh, 'sha256').hexdigest())
|
||||
return sums
|
||||
|
||||
|
||||
def sha256_by_suffix(sums: FileSums) -> Dict[str, str]:
|
||||
"""The hex sha256 of each file keyed by its suffix. The files of a FileGroup share a stem, so a suffix names one
|
||||
file even after the group is renamed."""
|
||||
return {split_path_stem_and_suffix(path)[1]: sha256 for path, (_, sha256) in sums.items()}
|
||||
|
||||
|
||||
refresh_logger = logger.getChild('refresh')
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import zipfile
|
||||
|
|
@ -1988,3 +1990,33 @@ def test_directory_or_descendant(test_session):
|
|||
plan = ' '.join(row[3] for row in test_session.execute(sqlalchemy.text(f'EXPLAIN QUERY PLAN {statement}')))
|
||||
assert 'SCAN file_group' not in plan, plan
|
||||
assert 'file_group_directory_idx' in plan, plan
|
||||
|
||||
|
||||
def test_sha256_files(test_directory):
|
||||
"""Each file is hashed; the metadata JSON, which holds the sums, and missing files are skipped."""
|
||||
video, caption = test_directory / 'foo.mp4', test_directory / 'foo.en.vtt'
|
||||
video.write_bytes(b'video')
|
||||
caption.write_bytes(b'caption')
|
||||
(test_directory / 'foo.info.json').write_text('{}')
|
||||
(test_directory / 'foo.readability.json').write_text('{}')
|
||||
paths = [video, caption, test_directory / 'foo.info.json', test_directory / 'foo.readability.json',
|
||||
test_directory / 'foo.webp']
|
||||
|
||||
sums = lib.sha256_files(paths)
|
||||
assert sums == {
|
||||
video: (video.stat().st_mtime_ns, hashlib.sha256(b'video').hexdigest()),
|
||||
caption: (caption.stat().st_mtime_ns, hashlib.sha256(b'caption').hexdigest()),
|
||||
}
|
||||
# Keyed by suffix: every file of a FileGroup shares a stem, which a rename changes.
|
||||
assert lib.sha256_by_suffix(sums) == {
|
||||
'.mp4': hashlib.sha256(b'video').hexdigest(),
|
||||
'.en.vtt': hashlib.sha256(b'caption').hexdigest(),
|
||||
}
|
||||
|
||||
# A known sum is reused while the file is unchanged.
|
||||
known = {video: (video.stat().st_mtime_ns, 'known')}
|
||||
assert lib.sha256_files([video], known=known) == known
|
||||
# A file changed since it was hashed is hashed again.
|
||||
video.write_bytes(b'video changed')
|
||||
os.utime(video, ns=(0, 0))
|
||||
assert lib.sha256_files([video], known=known) == {video: (0, hashlib.sha256(b'video changed').hexdigest())}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue