From 32405a913249c560705acbdb68195a6aebd5fc36 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:34:49 -0600 Subject: [PATCH] A download records the sha256 of its files in the wrolpi section (#904) * A download records the sha256 of its files in the wrolpi section A video download writes `wrolpi.sha256` into its .info.json; an archive download writes it into its .readability.json. Sums are keyed by suffix, so a rename or Reorganize keeps them valid, and the metadata JSON holding them is not hashed. Large files are hashed in execute_download (in a thread, without the DB session). Finalize reuses those sums while a file's mtime is unchanged and hashes only files finalize created or changed (a converted poster, the ffprobe json). Co-Authored-By: Claude Opus 5.5 * A video download hashes only for an info json, and finalize never suspends Finalize holds the write session; awaiting the hash let the loop run a switch whose write waited on that session, stalling the loop for the busy timeout. Finalize now hashes inline: the large files were hashed in execute_download. A download that writes no info json has nowhere to keep the sums, so it no longer reads its files to hash them. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- modules/archive/__init__.py | 15 ++- .../archive/test/test_archive_downloader.py | 32 +++++ modules/videos/downloader.py | 22 +++- modules/videos/test/test_downloader.py | 116 +++++++++++++++++- wrolpi/files/lib.py | 33 +++++ wrolpi/files/test/test_lib.py | 32 +++++ 6 files changed, 245 insertions(+), 5 deletions(-) diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 573882db..a07971fe 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -1,3 +1,4 @@ +import asyncio import json import os import pathlib @@ -17,6 +18,7 @@ from wrolpi.db import get_db_session from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult from wrolpi.errors import UnrecoverableDownloadError from wrolpi.files.modeler import run_modeler_loop, skip_clause, SkipModeler +from wrolpi.files.lib import sha256_files, sha256_by_suffix, FileSums from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST, DOCKERIZED, DOWNLOAD_USER_AGENT, PROJECT_DIR from . import lib @@ -48,6 +50,7 @@ class PreparedArchive: class ExecutedArchive: """Output of ArchiveDownloader.execute_download: files are on disk and ready to register.""" written: WrittenArchiveFiles + sha256: FileSums = None # sums of the written files, hashed without the DB session class ArchiveDownloader(Downloader, ABC): @@ -98,11 +101,13 @@ class ArchiveDownloader(Downloader, ABC): written = write_archive_files(artifacts.url, artifacts.singlefile, artifacts.readability, artifacts.screenshot, destination=prepared.destination) - return ExecutedArchive(written=written) + sha256 = await asyncio.to_thread(sha256_files, written.paths) + return ExecutedArchive(written=written, sha256=sha256) def finalize_download(self, session: Session, download: Download, executed: ExecutedArchive) -> DownloadResult: - """Register the Archive record, apply tags, persist parent_download_url. Caller commits.""" + """Register the Archive record, apply tags, persist parent_download_url and the sha256 of its files. + Caller commits.""" archive = register_archive(session, executed.written) if tag_names := download.tag_names: @@ -116,6 +121,12 @@ class ArchiveDownloader(Downloader, ABC): except ValueError as e: logger.warning(f'Failed to update wrolpi json for {archive}: {e}') + try: + sums = sha256_files(archive.file_group.my_paths(), executed.sha256) + archive.file_group.update_wrolpi_json({'sha256': sha256_by_suffix(sums)}) + except (OSError, ValueError) as e: + logger.warning(f'Failed to record sha256 of {archive}', exc_info=e) + logger.info(f'Successfully downloaded Archive {download.url} {archive}') return DownloadResult(success=True, location=archive.location) diff --git a/modules/archive/test/test_archive_downloader.py b/modules/archive/test/test_archive_downloader.py index c7171703..b4868605 100644 --- a/modules/archive/test/test_archive_downloader.py +++ b/modules/archive/test/test_archive_downloader.py @@ -5,10 +5,13 @@ isolation. None of them use async_client or test_download_manager — that's th whole point of the refactor. """ import asyncio +import hashlib +import json import pathlib import pytest +import modules.archive from modules.archive import ( ExecutedArchive, PreparedArchive, @@ -266,3 +269,32 @@ async def test_do_singlefile_configured_browser(test_session, test_directory, mo capture, = calls assert capture['env']['WROLPI_BROWSER'] == expected + + +@pytest.mark.asyncio +async def test_download_records_sha256(async_client, test_session, test_directory, monkeypatch): + """An archive download records the sha256 of each file in its group in the readability json's wrolpi section.""" + singlefile, readability, screenshot = make_fake_archive_result() + + async def mock_request_archive(url, compress=False): + return singlefile, readability, screenshot + + monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive) + + download = Download(url='https://example.com/x', downloader='archive', + settings={'parent_download_url': 'https://example.com/feed'}) + prepared = archive_downloader.prepare_download(test_session, download) + executed = await archive_downloader.execute_download(prepared, make_test_ctx()) + result = archive_downloader.finalize_download(test_session, download, executed) + test_session.commit() + assert result.success is True + + archive = test_session.query(Archive).one() + readability_json_path = archive.file_group.readability_json_path + wrolpi = json.loads(readability_json_path.read_text())['wrolpi'] + assert wrolpi['parent_download_url'] == 'https://example.com/feed' + stem = archive.singlefile_path.name[:-len('.html')] + expected = {path.name[len(stem):]: hashlib.sha256(path.read_bytes()).hexdigest() + for path in archive.file_group.my_paths() if path != readability_json_path} + assert set(expected) == {'.html', '.readability.html', '.readability.txt', '.png'} + assert wrolpi['sha256'] == expected diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 82ff2be8..423f8963 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -1,4 +1,5 @@ #! /usr/bin/env python3 +import asyncio import copy import dataclasses import json @@ -30,7 +31,8 @@ from wrolpi.db import get_db_session from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \ clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError -from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix +from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix, sha256_files, sha256_by_suffix, \ + FileSums from wrolpi.files.models import FileGroup from wrolpi.files.worker import file_worker from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR @@ -923,6 +925,7 @@ class ExecutedVideo: settings: dict location: Optional[str] # for error reporting wrolpi: dict = field(default_factory=dict) # existing video's wrolpi section, to carry over + sha256: FileSums = field(default_factory=dict) # sums of video_paths, hashed without the DB session class VideoDownloader(Downloader, ABC): @@ -1209,6 +1212,12 @@ class VideoDownloader(Downloader, ABC): except Exception: logger.warning(f'Failed to center YouTube captions in {path}', exc_info=True) + # Hash the (large) files here; finalize holds the DB session. The sums are kept in the info json, so + # without one there is nothing to hash for. + sha256 = dict() + if effective['writeinfojson'] or any(i.name.endswith('.info.json') for i in video_paths): + sha256 = await asyncio.to_thread(sha256_files, video_paths) + return ExecutedVideo( url=url, video_path=video_path, @@ -1221,6 +1230,7 @@ class VideoDownloader(Downloader, ABC): settings=settings, location=location, wrolpi=prepared.wrolpi, + sha256=sha256, ) except BotBlockedDownloadError: @@ -1300,6 +1310,16 @@ class VideoDownloader(Downloader, ABC): error='Download did not contain audio stream', location=location) + # Hashed once the files are final: validation may have converted the poster, or written the ffprobe json. + # Not awaited: this session holds the write lock, so suspending would let a switch's write stall the loop. + # Only files changed since execute_download are read, and those are small. + try: + if video.info_json_path: + sums = sha256_files(video.file_group.my_paths(), executed.sha256) + video.file_group.update_wrolpi_json({'sha256': sha256_by_suffix(sums)}) + except (OSError, ValueError) as e: + logger.warning(f'Failed to record sha256 of {video}', exc_info=e) + logger.info(f'Successfully downloaded video {executed.url} {video}') # Delete duplicate Videos that share the URL or source_id. Runs against the same diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 40084e2d..038393e9 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,4 +1,5 @@ import asyncio +import hashlib import json import shutil from copy import copy @@ -10,7 +11,8 @@ import pytest from modules.videos.downloader import VideoDownloader, \ get_or_create_channel, channel_downloader, video_downloader, preview_filename, \ prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \ - parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP + parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP, \ + ExecutedVideo from modules.videos.lib import get_videos_downloader_config from modules.videos.models import Channel, Video from wrolpi.conftest import test_directory, await_switches @@ -1992,5 +1994,115 @@ async def test_redownload_keeps_wrolpi_section(test_session, test_directory, moc video: Video = test_session.query(Video).one() info_json = video.get_info_json() assert info_json['title'] == 'Orig v2' - assert info_json['wrolpi'] == {'custom_title': 'Mine'} + ffprobe_json_path = video_path.with_suffix('.ffprobe.json') + assert info_json['wrolpi'] == { + 'custom_title': 'Mine', + 'sha256': {'.mp4': hashlib.sha256(video_path.read_bytes()).hexdigest(), + '.ffprobe.json': hashlib.sha256(ffprobe_json_path.read_bytes()).hexdigest()}, + } assert video.file_group.title == 'Mine' + + +@pytest.mark.asyncio +async def test_download_records_sha256(test_session, test_directory, mock_video_extract_info, simple_channel, + await_switches, video_download_manager, mock_video_process_runner, + image_bytes_factory): + """A download records the sha256 of each file in its group, as the files are once the download finishes.""" + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + simple_channel.generate_posters = True + test_session.commit() + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + video_path = simple_channel.directory / 'a video.mp4' + info_json_path = video_path.with_suffix('.info.json') + + from wrolpi.cmd import CommandResult + + async def yt_dlp_writes_files(*args, **kwargs): + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path) + shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', video_path.with_suffix('.en.vtt')) + # A WebP poster, which is converted to a JPEG once downloaded. + video_path.with_suffix('.webp').write_bytes(image_bytes_factory('.webp')) + info_json_path.write_text(json.dumps({'title': 'Title', 'webpage_url': url, 'duration': 5})) + return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0) + + mock_video_process_runner.side_effect = yt_dlp_writes_files + with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename: + mock_video_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + video_download_manager.create_download(test_session, url, video_downloader.name) + await video_download_manager.wait_for_all_downloads() + + test_session.expire_all() + video: Video = test_session.query(Video).one() + sums = video.get_info_json()['wrolpi']['sha256'] + assert not video_path.with_suffix('.webp').exists() + assert '.webp' not in sums, 'the converted poster is gone' + assert '.info.json' not in sums, 'the info json holds the sums' + expected = {path.name[len('a video'):]: hashlib.sha256(path.read_bytes()).hexdigest() + for path in video.file_group.my_paths() if path != info_json_path and path.is_file()} + assert {'.mp4', '.en.vtt', '.jpg'} <= set(expected) + assert sums == expected + + +@pytest.mark.asyncio +async def test_finalize_never_suspends(test_session, test_directory): + """finalize_download holds the write session from start to end. Were it to suspend, the event loop could run a + switch whose (blocking) write waits on that session, and the loop would stall until the database's busy timeout. + So once the files are on disk, finalize completes without suspending.""" + directory = test_directory / 'videos/channel name' + directory.mkdir(parents=True) + video_path = directory / 'a video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path) + info_json_path = video_path.with_suffix('.info.json') + url = 'https://example.com/the-video' + info_json_path.write_text(json.dumps({'id': 'the-id', 'title': 'Title', 'webpage_url': url})) + executed = ExecutedVideo(url=url, video_path=video_path, video_paths=[video_path, info_json_path], + info_json=dict(), entry={'id': 'the-id'}, channel_id=None, audio_only=False, + tag_names=[], settings=dict(), location=None) + + coro = video_downloader.finalize_download(test_session, Download(url=url, downloader='video'), executed) + try: + coro.send(None) + except StopIteration as e: + result = e.value + else: + coro.close() + pytest.fail('finalize_download suspended') + + assert result.success is True, result + sums = json.loads(info_json_path.read_text())['wrolpi']['sha256'] + assert sums['.mp4'] == hashlib.sha256(video_path.read_bytes()).hexdigest() + + +@pytest.mark.asyncio +async def test_download_without_info_json_skips_sha256(test_session, test_directory, mock_video_extract_info, + simple_channel, await_switches, video_download_manager, + mock_video_process_runner): + """Sums are kept in the info json; a download that writes none does not read its files to hash them.""" + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + test_session.commit() + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + video_path = simple_channel.directory / 'a video.mp4' + + from wrolpi.cmd import CommandResult + + async def yt_dlp_writes_video(*args, **kwargs): + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path) + return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0) + + mock_video_process_runner.side_effect = yt_dlp_writes_video + with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename, \ + mock.patch('modules.videos.downloader.sha256_files') as mock_sha256_files: + mock_video_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + video_download_manager.create_download(test_session, url, video_downloader.name, + settings={'writeinfojson': False}) + await video_download_manager.wait_for_all_downloads() + + video: Video = test_session.query(Video).one() + assert video.info_json_path is None + mock_sha256_files.assert_not_called() diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 94804b2a..99369c08 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -4,6 +4,7 @@ import dataclasses import datetime import functools import glob +import hashlib import json import mimetypes import os @@ -692,6 +693,38 @@ def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ... return results +# The metadata JSON holds the sums, so it cannot hold its own. +METADATA_JSON_SUFFIXES = ('.info.json', '.readability.json') + +# A file's (modification time in nanoseconds, hex sha256), keyed by its path. +FileSums = Dict[pathlib.Path, Tuple[int, str]] + + +def sha256_files(paths: Iterable[pathlib.Path], known: FileSums = None) -> FileSums: + """Hash each file, skipping the metadata JSON and files that do not exist. + + A `known` sum is reused while its file's modification time is unchanged. This reads whole files; call it in a + thread.""" + known = known or dict() + sums = dict() + for path in paths: + if path.name.endswith(METADATA_JSON_SUFFIXES) or not path.is_file(): + continue + mtime_ns = path.stat().st_mtime_ns + if (known_sum := known.get(path)) and known_sum[0] == mtime_ns: + sums[path] = known_sum + continue + with path.open('rb') as fh: + sums[path] = (mtime_ns, hashlib.file_digest(fh, 'sha256').hexdigest()) + return sums + + +def sha256_by_suffix(sums: FileSums) -> Dict[str, str]: + """The hex sha256 of each file keyed by its suffix. The files of a FileGroup share a stem, so a suffix names one + file even after the group is renamed.""" + return {split_path_stem_and_suffix(path)[1]: sha256 for path, (_, sha256) in sums.items()} + + refresh_logger = logger.getChild('refresh') diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 639c33bc..2093aad7 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1,5 +1,7 @@ +import hashlib import itertools import json +import os import pathlib import shutil import zipfile @@ -1988,3 +1990,33 @@ def test_directory_or_descendant(test_session): plan = ' '.join(row[3] for row in test_session.execute(sqlalchemy.text(f'EXPLAIN QUERY PLAN {statement}'))) assert 'SCAN file_group' not in plan, plan assert 'file_group_directory_idx' in plan, plan + + +def test_sha256_files(test_directory): + """Each file is hashed; the metadata JSON, which holds the sums, and missing files are skipped.""" + video, caption = test_directory / 'foo.mp4', test_directory / 'foo.en.vtt' + video.write_bytes(b'video') + caption.write_bytes(b'caption') + (test_directory / 'foo.info.json').write_text('{}') + (test_directory / 'foo.readability.json').write_text('{}') + paths = [video, caption, test_directory / 'foo.info.json', test_directory / 'foo.readability.json', + test_directory / 'foo.webp'] + + sums = lib.sha256_files(paths) + assert sums == { + video: (video.stat().st_mtime_ns, hashlib.sha256(b'video').hexdigest()), + caption: (caption.stat().st_mtime_ns, hashlib.sha256(b'caption').hexdigest()), + } + # Keyed by suffix: every file of a FileGroup shares a stem, which a rename changes. + assert lib.sha256_by_suffix(sums) == { + '.mp4': hashlib.sha256(b'video').hexdigest(), + '.en.vtt': hashlib.sha256(b'caption').hexdigest(), + } + + # A known sum is reused while the file is unchanged. + known = {video: (video.stat().st_mtime_ns, 'known')} + assert lib.sha256_files([video], known=known) == known + # A file changed since it was hashed is hashed again. + video.write_bytes(b'video changed') + os.utime(video, ns=(0, 0)) + assert lib.sha256_files([video], known=known) == {video: (0, hashlib.sha256(b'video changed').hexdigest())}