Updating archive downloader to use download.destination.

This commit is contained in:
Roland 2025-12-17 16:56:58 -07:00
parent 0ea626c007
commit a8ff1a1c38
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
6 changed files with 181 additions and 31 deletions

View file

@ -44,15 +44,21 @@ class ArchiveDownloader(Downloader, ABC):
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
# Get destination from settings (passed by RSS downloader from download.destination column)
destination = None
if download.settings and download.settings.get('destination'):
destination = pathlib.Path(download.settings['destination'])
if DOCKERIZED or PYTEST:
# Perform the archive in the Archive docker container. (Typically in the development environment).
singlefile, readability, screenshot = await request_archive(download.url)
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
archive: Archive = await lib.model_archive_result(
download.url, singlefile, readability, screenshot, destination=destination)
archive_id = archive.id
else:
# Perform the archive using locally installed executables.
singlefile = await self.do_singlefile(download)
archive = await lib.singlefile_to_archive(singlefile)
archive = await lib.singlefile_to_archive(singlefile, destination=destination)
archive_id = archive.id
with get_db_session() as session:
@ -211,27 +217,9 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive:
archive.validate()
archive.flush()
# Validate that archive path matches domain directory (efficient O(1) check)
if archive.collection_id and archive.collection:
collection = archive.collection
if collection.kind == 'domain' and collection.directory and not collection.tag_id:
# Check if archive path is under the domain's directory
archive_path = pathlib.Path(file_group.primary_path)
from wrolpi.common import get_media_directory
media_dir = get_media_directory()
collection_abs_dir = media_dir / collection.directory
try:
# Check if archive is under the collection's directory
archive_path.relative_to(collection_abs_dir)
except ValueError:
# Archive is NOT under the domain's directory - clear it
logger.warning(
f'Archive {archive.id} at {archive_path} is not under domain directory {collection.directory}. '
f'Clearing domain directory (domain is not tagged).'
)
collection.directory = None
session.flush([collection])
# Note: We intentionally do NOT clear collection.directory when archive is outside it.
# Archives are associated with domain collections based on URL, not file location.
# The collection's directory should remain intact for other archives.
file_group.title = file_group.a_text = title or archive.file_group.title
file_group.d_text = contents

View file

@ -277,9 +277,19 @@ def get_archive_destination(domain_collection: 'Collection') -> pathlib.Path:
MAXIMUM_ARCHIVE_FILE_CHARACTER_LENGTH = 200
def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles:
"""Create a list of archive files using a shared name schema. Raise an error if any of them exist."""
directory = get_domain_directory(url)
def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.Path = None) -> ArchiveFiles:
"""Create a list of archive files using a shared name schema. Raise an error if any of them exist.
Args:
url: The URL being archived (used to derive domain if no destination)
title: Title for the archive files
destination: Optional directory to save files to. If provided, files go here instead of archive/<domain>
"""
if destination:
directory = destination
directory.mkdir(parents=True, exist_ok=True)
else:
directory = get_domain_directory(url)
# Datetime is valid in Linux and Windows.
dt = archive_strftime(now())
@ -391,9 +401,17 @@ async def request_screenshot(url: str, singlefile_path: pathlib.Path) -> Optiona
return screenshot
async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes) -> Archive:
async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes,
destination: pathlib.Path = None) -> Archive:
"""
Convert results from ArchiveDownloader into real files. Create Archive record.
Args:
url: The URL that was archived
singlefile: The HTML content from SingleFile
readability: The readability extraction results
screenshot: The screenshot bytes
destination: Optional directory to save files to. If provided, files go here instead of archive/<domain>
"""
readability = readability.copy() if readability else readability
# First try to get the title from Readability.
@ -406,7 +424,7 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
# Readability could not find title, lets use ours.
readability['title'] = title
archive_files = get_new_archive_files(url, title)
archive_files = get_new_archive_files(url, title, destination=destination)
if readability:
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
@ -1114,11 +1132,15 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
raise RuntimeError(f'Failed to extract readability for {url} got {result.return_code}') from e
async def singlefile_to_archive(singlefile: bytes) -> Archive:
async def singlefile_to_archive(singlefile: bytes, destination: pathlib.Path = None) -> Archive:
"""
Convert a SingleFile to an Archive.
This is done by extracting readability, creating a screenshot, then attaching them to an Archive/FileGroup.
Args:
singlefile: The SingleFile HTML bytes
destination: Optional directory to save files to. If provided, files go here instead of archive/<domain>
"""
# Get URL first because it does some simple checking of `singlefile`
url = get_url_from_singlefile(singlefile)
@ -1146,7 +1168,7 @@ async def singlefile_to_archive(singlefile: bytes) -> Archive:
logger.error(f'Failed to extract screenshot from: {url}', exc_info=e)
logger.trace(f'singlefile_to_archive modeling: {url}')
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot)
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot, destination=destination)
return archive

View file

@ -138,3 +138,65 @@ async def test_model_archive_extracts_url_from_singlefile(async_client, test_ses
collection = test_session.query(Collection).get(archive.collection_id)
assert collection.name == domain
assert collection.kind == 'domain'
@pytest.mark.asyncio
async def test_archive_outside_directory_keeps_collection_directory(async_client, test_session, test_directory,
make_files_structure):
"""
Test that model_archive() does NOT clear collection.directory when archive is outside it.
Bug: Previously, if an archive was found outside its domain collection's directory,
the code would clear collection.directory = None. This was wrong because:
1. Archives should still be associated with domain collection based on URL
2. The collection's directory should remain intact for other archives
"""
import datetime
import pytz
domain = 'zerohedge.com'
url = f'https://{domain}/article'
# Create a domain collection WITH a specific directory
collection_dir = 'archive/News/zerohedge.com'
collection = Collection(name=domain, kind='domain', directory=collection_dir)
test_session.add(collection)
test_session.commit()
collection_id = collection.id
# Create the collection directory
(test_directory / collection_dir).mkdir(parents=True, exist_ok=True)
# Create archive files in a DIFFERENT directory (simulating manual file move)
old_archive_dir = test_directory / 'archive' / domain
old_archive_dir.mkdir(parents=True)
timestamp = archive_strftime(datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC))
title = 'Test Article'
singlefile_content = f'''<!DOCTYPE html><html><!--
Page saved with SingleFile
url: {url}
--><head><title>{title}</title></head>
<body><p>Content</p></body></html>'''
files = make_files_structure({
str(old_archive_dir / f'{timestamp}_{title}.html'): singlefile_content.strip(),
str(old_archive_dir / f'{timestamp}_{title}.readability.json'): '{"title": "' + title + '"}',
})
from wrolpi.files.models import FileGroup
file_paths = [pathlib.Path(f) for f in files]
file_group = FileGroup.from_paths(test_session, *file_paths)
# Model the archive - archive is in archive/zerohedge.com but collection dir is archive/News/zerohedge.com
archive = model_archive(test_session, file_group)
assert archive is not None
assert archive.collection_id == collection_id, "Archive should be associated with domain collection"
# CRITICAL: The collection's directory should NOT be cleared
test_session.refresh(collection)
assert str(collection.directory) == collection_dir, \
f"Collection directory should NOT be cleared! Expected '{collection_dir}', got '{collection.directory}'"

View file

@ -393,6 +393,25 @@ def test_get_new_archive_files(fake_now):
assert str(archive_files.screenshot).endswith('archive/example.com/2001-01-01-00-00-00_Title.png')
@skip_circleci
def test_get_new_archive_files_with_destination(test_directory, fake_now):
"""Archive files can be created in a custom destination directory instead of the default."""
fake_now(datetime(2001, 1, 1))
custom_destination = test_directory / 'archive/News/custom-domain.com'
custom_destination.mkdir(parents=True, exist_ok=True)
# When destination is provided, files should go there instead of archive/<domain>
archive_files = get_new_archive_files('https://example.com/page', 'Title', destination=custom_destination)
# Files should be in the custom destination, not archive/example.com
assert str(archive_files.singlefile).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.html')
assert str(archive_files.readability).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.html')
assert str(archive_files.readability_txt).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.txt')
assert str(archive_files.readability_json).endswith(
'archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.json')
assert str(archive_files.screenshot).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.png')
@skip_circleci
@pytest.mark.asyncio
async def test_title_in_filename(test_session, fake_now, test_directory, image_bytes_factory):

View file

@ -1509,7 +1509,11 @@ class RSSDownloader(Downloader):
next_download_settings['video_resolutions'] = i
if i := settings.get('video_format'):
next_download_settings['video_format'] = i
if i := settings.get('destination'):
# Use download.destination column (settings['destination'] is legacy)
if download.destination:
next_download_settings['destination'] = str(download.destination)
elif i := settings.get('destination'):
# Fallback to legacy settings for backwards compatibility
next_download_settings['destination'] = i
result = DownloadResult(

View file

@ -123,3 +123,58 @@ async def test_rss_downloader_filter_titles(test_session):
dict(link='https://example.com/a', title='A'),
dict(link='https://example.com/b', title='b'),
]
@pytest.mark.asyncio
async def test_rss_downloader_uses_destination_column(test_session, test_download_manager, await_switches):
"""RSS downloader should pass download.destination to child downloads, not settings['destination'].
The destination column is the source of truth. settings['destination'] is legacy.
"""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
http_downloader = RSSHTTPDownloader()
test_download_manager.register_downloader(http_downloader)
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(
bozo=0,
entries=[
dict(link='https://example.com/a'),
dict(link='https://example.com/b'),
]
)
# Create RSS download with destination column set, but empty settings.
test_download_manager.create_download(
test_session,
'https://example.com/feed',
rss_downloader.name,
sub_downloader_name='rss_http',
destination='archive/custom/path', # destination column
settings={}, # Empty settings - destination should come from column
)
await test_download_manager.wait_for_all_downloads()
# Get all downloads
downloads = test_session.query(Download).order_by(Download.id).all()
assert len(downloads) == 3, 'Expected 1 RSS download + 2 child downloads'
# The RSS download itself
rss_download = downloads[0]
assert rss_download.url == 'https://example.com/feed'
assert str(rss_download.destination).endswith('archive/custom/path')
# Child downloads should have destination in their settings (passed from RSS download.destination)
child_a = downloads[1]
child_b = downloads[2]
assert child_a.url == 'https://example.com/a'
assert child_b.url == 'https://example.com/b'
# The destination should be passed from RSS download.destination column to child settings
assert child_a.settings.get('destination') is not None, \
'Child download should inherit destination from RSS download.destination column'
assert child_a.settings['destination'].endswith('archive/custom/path'), \
f"Expected destination ending with 'archive/custom/path', got {child_a.settings.get('destination')}"
assert child_b.settings.get('destination') is not None, \
'Child download should inherit destination from RSS download.destination column'
assert child_b.settings['destination'].endswith('archive/custom/path'), \
f"Expected destination ending with 'archive/custom/path', got {child_b.settings.get('destination')}"