A download records the sha256 of its files in the wrolpi section (#904)

* A download records the sha256 of its files in the wrolpi section

A video download writes `wrolpi.sha256` into its .info.json; an archive download writes it into
its .readability.json.  Sums are keyed by suffix, so a rename or Reorganize keeps them valid,
and the metadata JSON holding them is not hashed.

Large files are hashed in execute_download (in a thread, without the DB session).  Finalize
reuses those sums while a file's mtime is unchanged and hashes only files finalize created or
changed (a converted poster, the ffprobe json).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* A video download hashes only for an info json, and finalize never suspends

Finalize holds the write session; awaiting the hash let the loop run a switch whose
write waited on that session, stalling the loop for the busy timeout.  Finalize now
hashes inline: the large files were hashed in execute_download.

A download that writes no info json has nowhere to keep the sums, so it no longer
reads its files to hash them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-30 19:34:49 -06:00 • committed by GitHub
parent a40beac0f6
commit 32405a9132
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 245 additions and 5 deletions

View file

@ -1,3 +1,4 @@
import asyncio
import json
import os
import pathlib
@ -17,6 +18,7 @@ from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.modeler import run_modeler_loop, skip_clause, SkipModeler
from wrolpi.files.lib import sha256_files, sha256_by_suffix, FileSums
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, DOCKERIZED, DOWNLOAD_USER_AGENT, PROJECT_DIR
from . import lib
@ -48,6 +50,7 @@ class PreparedArchive:
class ExecutedArchive:
"""Output of ArchiveDownloader.execute_download: files are on disk and ready to register."""
written: WrittenArchiveFiles
sha256: FileSums = None # sums of the written files, hashed without the DB session
class ArchiveDownloader(Downloader, ABC):
@ -98,11 +101,13 @@ class ArchiveDownloader(Downloader, ABC):
written = write_archive_files(artifacts.url, artifacts.singlefile, artifacts.readability,
artifacts.screenshot, destination=prepared.destination)
return ExecutedArchive(written=written)
sha256 = await asyncio.to_thread(sha256_files, written.paths)
return ExecutedArchive(written=written, sha256=sha256)
def finalize_download(self, session: Session, download: Download,
executed: ExecutedArchive) -> DownloadResult:
"""Register the Archive record, apply tags, persist parent_download_url. Caller commits."""
"""Register the Archive record, apply tags, persist parent_download_url and the sha256 of its files.
Caller commits."""
archive = register_archive(session, executed.written)
if tag_names := download.tag_names:
@ -116,6 +121,12 @@ class ArchiveDownloader(Downloader, ABC):
except ValueError as e:
logger.warning(f'Failed to update wrolpi json for {archive}: {e}')
try:
sums = sha256_files(archive.file_group.my_paths(), executed.sha256)
archive.file_group.update_wrolpi_json({'sha256': sha256_by_suffix(sums)})
except (OSError, ValueError) as e:
logger.warning(f'Failed to record sha256 of {archive}', exc_info=e)
logger.info(f'Successfully downloaded Archive {download.url} {archive}')
return DownloadResult(success=True, location=archive.location)

View file

@ -5,10 +5,13 @@ isolation. None of them use async_client or test_download_manager — that's th
whole point of the refactor.
"""
import asyncio
import hashlib
import json
import pathlib
import pytest
import modules.archive
from modules.archive import (
ExecutedArchive,
PreparedArchive,
@ -266,3 +269,32 @@ async def test_do_singlefile_configured_browser(test_session, test_directory, mo
capture, = calls
assert capture['env']['WROLPI_BROWSER'] == expected
@pytest.mark.asyncio
async def test_download_records_sha256(async_client, test_session, test_directory, monkeypatch):
"""An archive download records the sha256 of each file in its group in the readability json's wrolpi section."""
singlefile, readability, screenshot = make_fake_archive_result()
async def mock_request_archive(url, compress=False):
return singlefile, readability, screenshot
monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)
download = Download(url='https://example.com/x', downloader='archive',
settings={'parent_download_url': 'https://example.com/feed'})
prepared = archive_downloader.prepare_download(test_session, download)
executed = await archive_downloader.execute_download(prepared, make_test_ctx())
result = archive_downloader.finalize_download(test_session, download, executed)
test_session.commit()
assert result.success is True
archive = test_session.query(Archive).one()
readability_json_path = archive.file_group.readability_json_path
wrolpi = json.loads(readability_json_path.read_text())['wrolpi']
assert wrolpi['parent_download_url'] == 'https://example.com/feed'
stem = archive.singlefile_path.name[:-len('.html')]
expected = {path.name[len(stem):]: hashlib.sha256(path.read_bytes()).hexdigest()
for path in archive.file_group.my_paths() if path != readability_json_path}
assert set(expected) == {'.html', '.readability.html', '.readability.txt', '.png'}
assert wrolpi['sha256'] == expected

View file

@ -1,4 +1,5 @@
#! /usr/bin/env python3
import asyncio
import copy
import dataclasses
import json
@ -30,7 +31,8 @@ from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \
clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix, sha256_files, sha256_by_suffix, \
FileSums
from wrolpi.files.models import FileGroup
from wrolpi.files.worker import file_worker
from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
@ -923,6 +925,7 @@ class ExecutedVideo:
settings: dict
location: Optional[str] # for error reporting
wrolpi: dict = field(default_factory=dict) # existing video's wrolpi section, to carry over
sha256: FileSums = field(default_factory=dict) # sums of video_paths, hashed without the DB session
class VideoDownloader(Downloader, ABC):
@ -1209,6 +1212,12 @@ class VideoDownloader(Downloader, ABC):
except Exception:
logger.warning(f'Failed to center YouTube captions in {path}', exc_info=True)
# Hash the (large) files here; finalize holds the DB session. The sums are kept in the info json, so
# without one there is nothing to hash for.
sha256 = dict()
if effective['writeinfojson'] or any(i.name.endswith('.info.json') for i in video_paths):
sha256 = await asyncio.to_thread(sha256_files, video_paths)
return ExecutedVideo(
url=url,
video_path=video_path,
@ -1221,6 +1230,7 @@ class VideoDownloader(Downloader, ABC):
settings=settings,
location=location,
wrolpi=prepared.wrolpi,
sha256=sha256,
)
except BotBlockedDownloadError:
@ -1300,6 +1310,16 @@ class VideoDownloader(Downloader, ABC):
error='Download did not contain audio stream',
location=location)
# Hashed once the files are final: validation may have converted the poster, or written the ffprobe json.
# Not awaited: this session holds the write lock, so suspending would let a switch's write stall the loop.
# Only files changed since execute_download are read, and those are small.
try:
if video.info_json_path:
sums = sha256_files(video.file_group.my_paths(), executed.sha256)
video.file_group.update_wrolpi_json({'sha256': sha256_by_suffix(sums)})
except (OSError, ValueError) as e:
logger.warning(f'Failed to record sha256 of {video}', exc_info=e)
logger.info(f'Successfully downloaded video {executed.url} {video}')
# Delete duplicate Videos that share the URL or source_id. Runs against the same

View file

@ -1,4 +1,5 @@
import asyncio
import hashlib
import json
import shutil
from copy import copy
@ -10,7 +11,8 @@ import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \
parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP
parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP, \
ExecutedVideo
from modules.videos.lib import get_videos_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
@ -1992,5 +1994,115 @@ async def test_redownload_keeps_wrolpi_section(test_session, test_directory, moc
video: Video = test_session.query(Video).one()
info_json = video.get_info_json()
assert info_json['title'] == 'Orig v2'
assert info_json['wrolpi'] == {'custom_title': 'Mine'}
ffprobe_json_path = video_path.with_suffix('.ffprobe.json')
assert info_json['wrolpi'] == {
'custom_title': 'Mine',
'sha256': {'.mp4': hashlib.sha256(video_path.read_bytes()).hexdigest(),
'.ffprobe.json': hashlib.sha256(ffprobe_json_path.read_bytes()).hexdigest()},
}
assert video.file_group.title == 'Mine'
@pytest.mark.asyncio
async def test_download_records_sha256(test_session, test_directory, mock_video_extract_info, simple_channel,
await_switches, video_download_manager, mock_video_process_runner,
image_bytes_factory):
"""A download records the sha256 of each file in its group, as the files are once the download finishes."""
simple_channel.source_id = example_video_json['channel_id']
simple_channel.directory = test_directory / 'videos/channel name'
simple_channel.directory.mkdir(parents=True)
simple_channel.generate_posters = True
test_session.commit()
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
video_path = simple_channel.directory / 'a video.mp4'
info_json_path = video_path.with_suffix('.info.json')
from wrolpi.cmd import CommandResult
async def yt_dlp_writes_files(*args, **kwargs):
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', video_path.with_suffix('.en.vtt'))
# A WebP poster, which is converted to a JPEG once downloaded.
video_path.with_suffix('.webp').write_bytes(image_bytes_factory('.webp'))
info_json_path.write_text(json.dumps({'title': 'Title', 'webpage_url': url, 'duration': 5}))
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
mock_video_process_runner.side_effect = yt_dlp_writes_files
with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
mock_video_extract_info.return_value = example_video_json
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
video_download_manager.create_download(test_session, url, video_downloader.name)
await video_download_manager.wait_for_all_downloads()
test_session.expire_all()
video: Video = test_session.query(Video).one()
sums = video.get_info_json()['wrolpi']['sha256']
assert not video_path.with_suffix('.webp').exists()
assert '.webp' not in sums, 'the converted poster is gone'
assert '.info.json' not in sums, 'the info json holds the sums'
expected = {path.name[len('a video'):]: hashlib.sha256(path.read_bytes()).hexdigest()
for path in video.file_group.my_paths() if path != info_json_path and path.is_file()}
assert {'.mp4', '.en.vtt', '.jpg'} <= set(expected)
assert sums == expected
@pytest.mark.asyncio
async def test_finalize_never_suspends(test_session, test_directory):
"""finalize_download holds the write session from start to end. Were it to suspend, the event loop could run a
switch whose (blocking) write waits on that session, and the loop would stall until the database's busy timeout.
So once the files are on disk, finalize completes without suspending."""
directory = test_directory / 'videos/channel name'
directory.mkdir(parents=True)
video_path = directory / 'a video.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
info_json_path = video_path.with_suffix('.info.json')
url = 'https://example.com/the-video'
info_json_path.write_text(json.dumps({'id': 'the-id', 'title': 'Title', 'webpage_url': url}))
executed = ExecutedVideo(url=url, video_path=video_path, video_paths=[video_path, info_json_path],
info_json=dict(), entry={'id': 'the-id'}, channel_id=None, audio_only=False,
tag_names=[], settings=dict(), location=None)
coro = video_downloader.finalize_download(test_session, Download(url=url, downloader='video'), executed)
try:
coro.send(None)
except StopIteration as e:
result = e.value
else:
coro.close()
pytest.fail('finalize_download suspended')
assert result.success is True, result
sums = json.loads(info_json_path.read_text())['wrolpi']['sha256']
assert sums['.mp4'] == hashlib.sha256(video_path.read_bytes()).hexdigest()
@pytest.mark.asyncio
async def test_download_without_info_json_skips_sha256(test_session, test_directory, mock_video_extract_info,
simple_channel, await_switches, video_download_manager,
mock_video_process_runner):
"""Sums are kept in the info json; a download that writes none does not read its files to hash them."""
simple_channel.source_id = example_video_json['channel_id']
simple_channel.directory = test_directory / 'videos/channel name'
simple_channel.directory.mkdir(parents=True)
test_session.commit()
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
video_path = simple_channel.directory / 'a video.mp4'
from wrolpi.cmd import CommandResult
async def yt_dlp_writes_video(*args, **kwargs):
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
mock_video_process_runner.side_effect = yt_dlp_writes_video
with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.sha256_files') as mock_sha256_files:
mock_video_extract_info.return_value = example_video_json
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
video_download_manager.create_download(test_session, url, video_downloader.name,
settings={'writeinfojson': False})
await video_download_manager.wait_for_all_downloads()
video: Video = test_session.query(Video).one()
assert video.info_json_path is None
mock_sha256_files.assert_not_called()

View file

@ -4,6 +4,7 @@ import dataclasses
import datetime
import functools
import glob
import hashlib
import json
import mimetypes
import os
@ -692,6 +693,38 @@ def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...
return results
# The metadata JSON holds the sums, so it cannot hold its own.
METADATA_JSON_SUFFIXES = ('.info.json', '.readability.json')
# A file's (modification time in nanoseconds, hex sha256), keyed by its path.
FileSums = Dict[pathlib.Path, Tuple[int, str]]
def sha256_files(paths: Iterable[pathlib.Path], known: FileSums = None) -> FileSums:
"""Hash each file, skipping the metadata JSON and files that do not exist.
A `known` sum is reused while its file's modification time is unchanged. This reads whole files; call it in a
thread."""
known = known or dict()
sums = dict()
for path in paths:
if path.name.endswith(METADATA_JSON_SUFFIXES) or not path.is_file():
continue
mtime_ns = path.stat().st_mtime_ns
if (known_sum := known.get(path)) and known_sum[0] == mtime_ns:
sums[path] = known_sum
continue
with path.open('rb') as fh:
sums[path] = (mtime_ns, hashlib.file_digest(fh, 'sha256').hexdigest())
return sums
def sha256_by_suffix(sums: FileSums) -> Dict[str, str]:
"""The hex sha256 of each file keyed by its suffix. The files of a FileGroup share a stem, so a suffix names one
file even after the group is renamed."""
return {split_path_stem_and_suffix(path)[1]: sha256 for path, (_, sha256) in sums.items()}
refresh_logger = logger.getChild('refresh')

View file

@ -1,5 +1,7 @@
import hashlib
import itertools
import json
import os
import pathlib
import shutil
import zipfile
@ -1988,3 +1990,33 @@ def test_directory_or_descendant(test_session):
plan = ' '.join(row[3] for row in test_session.execute(sqlalchemy.text(f'EXPLAIN QUERY PLAN {statement}')))
assert 'SCAN file_group' not in plan, plan
assert 'file_group_directory_idx' in plan, plan
def test_sha256_files(test_directory):
"""Each file is hashed; the metadata JSON, which holds the sums, and missing files are skipped."""
video, caption = test_directory / 'foo.mp4', test_directory / 'foo.en.vtt'
video.write_bytes(b'video')
caption.write_bytes(b'caption')
(test_directory / 'foo.info.json').write_text('{}')
(test_directory / 'foo.readability.json').write_text('{}')
paths = [video, caption, test_directory / 'foo.info.json', test_directory / 'foo.readability.json',
test_directory / 'foo.webp']
sums = lib.sha256_files(paths)
assert sums == {
video: (video.stat().st_mtime_ns, hashlib.sha256(b'video').hexdigest()),
caption: (caption.stat().st_mtime_ns, hashlib.sha256(b'caption').hexdigest()),
}
# Keyed by suffix: every file of a FileGroup shares a stem, which a rename changes.
assert lib.sha256_by_suffix(sums) == {
'.mp4': hashlib.sha256(b'video').hexdigest(),
'.en.vtt': hashlib.sha256(b'caption').hexdigest(),
}
# A known sum is reused while the file is unchanged.
known = {video: (video.stat().st_mtime_ns, 'known')}
assert lib.sha256_files([video], known=known) == known
# A file changed since it was hashed is hashed again.
video.write_bytes(b'video changed')
os.utime(video, ns=(0, 0))
assert lib.sha256_files([video], known=known) == {video: (0, hashlib.sha256(b'video changed').hexdigest())}