mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Updating archive downloader to use download.destination.
This commit is contained in:
parent
0ea626c007
commit
a8ff1a1c38
6 changed files with 181 additions and 31 deletions
|
|
@ -44,15 +44,21 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
if download.attempts > 3:
|
||||
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
|
||||
|
||||
# Get destination from settings (passed by RSS downloader from download.destination column)
|
||||
destination = None
|
||||
if download.settings and download.settings.get('destination'):
|
||||
destination = pathlib.Path(download.settings['destination'])
|
||||
|
||||
if DOCKERIZED or PYTEST:
|
||||
# Perform the archive in the Archive docker container. (Typically in the development environment).
|
||||
singlefile, readability, screenshot = await request_archive(download.url)
|
||||
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
|
||||
archive: Archive = await lib.model_archive_result(
|
||||
download.url, singlefile, readability, screenshot, destination=destination)
|
||||
archive_id = archive.id
|
||||
else:
|
||||
# Perform the archive using locally installed executables.
|
||||
singlefile = await self.do_singlefile(download)
|
||||
archive = await lib.singlefile_to_archive(singlefile)
|
||||
archive = await lib.singlefile_to_archive(singlefile, destination=destination)
|
||||
archive_id = archive.id
|
||||
|
||||
with get_db_session() as session:
|
||||
|
|
@ -211,27 +217,9 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive:
|
|||
archive.validate()
|
||||
archive.flush()
|
||||
|
||||
# Validate that archive path matches domain directory (efficient O(1) check)
|
||||
if archive.collection_id and archive.collection:
|
||||
collection = archive.collection
|
||||
if collection.kind == 'domain' and collection.directory and not collection.tag_id:
|
||||
# Check if archive path is under the domain's directory
|
||||
archive_path = pathlib.Path(file_group.primary_path)
|
||||
from wrolpi.common import get_media_directory
|
||||
media_dir = get_media_directory()
|
||||
collection_abs_dir = media_dir / collection.directory
|
||||
|
||||
try:
|
||||
# Check if archive is under the collection's directory
|
||||
archive_path.relative_to(collection_abs_dir)
|
||||
except ValueError:
|
||||
# Archive is NOT under the domain's directory - clear it
|
||||
logger.warning(
|
||||
f'Archive {archive.id} at {archive_path} is not under domain directory {collection.directory}. '
|
||||
f'Clearing domain directory (domain is not tagged).'
|
||||
)
|
||||
collection.directory = None
|
||||
session.flush([collection])
|
||||
# Note: We intentionally do NOT clear collection.directory when archive is outside it.
|
||||
# Archives are associated with domain collections based on URL, not file location.
|
||||
# The collection's directory should remain intact for other archives.
|
||||
|
||||
file_group.title = file_group.a_text = title or archive.file_group.title
|
||||
file_group.d_text = contents
|
||||
|
|
|
|||
|
|
@ -277,9 +277,19 @@ def get_archive_destination(domain_collection: 'Collection') -> pathlib.Path:
|
|||
MAXIMUM_ARCHIVE_FILE_CHARACTER_LENGTH = 200
|
||||
|
||||
|
||||
def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles:
|
||||
"""Create a list of archive files using a shared name schema. Raise an error if any of them exist."""
|
||||
directory = get_domain_directory(url)
|
||||
def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.Path = None) -> ArchiveFiles:
|
||||
"""Create a list of archive files using a shared name schema. Raise an error if any of them exist.
|
||||
|
||||
Args:
|
||||
url: The URL being archived (used to derive domain if no destination)
|
||||
title: Title for the archive files
|
||||
destination: Optional directory to save files to. If provided, files go here instead of archive/<domain>
|
||||
"""
|
||||
if destination:
|
||||
directory = destination
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
directory = get_domain_directory(url)
|
||||
# Datetime is valid in Linux and Windows.
|
||||
dt = archive_strftime(now())
|
||||
|
||||
|
|
@ -391,9 +401,17 @@ async def request_screenshot(url: str, singlefile_path: pathlib.Path) -> Optiona
|
|||
return screenshot
|
||||
|
||||
|
||||
async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes) -> Archive:
|
||||
async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes,
|
||||
destination: pathlib.Path = None) -> Archive:
|
||||
"""
|
||||
Convert results from ArchiveDownloader into real files. Create Archive record.
|
||||
|
||||
Args:
|
||||
url: The URL that was archived
|
||||
singlefile: The HTML content from SingleFile
|
||||
readability: The readability extraction results
|
||||
screenshot: The screenshot bytes
|
||||
destination: Optional directory to save files to. If provided, files go here instead of archive/<domain>
|
||||
"""
|
||||
readability = readability.copy() if readability else readability
|
||||
# First try to get the title from Readability.
|
||||
|
|
@ -406,7 +424,7 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
|
|||
# Readability could not find title, lets use ours.
|
||||
readability['title'] = title
|
||||
|
||||
archive_files = get_new_archive_files(url, title)
|
||||
archive_files = get_new_archive_files(url, title, destination=destination)
|
||||
|
||||
if readability:
|
||||
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
|
||||
|
|
@ -1114,11 +1132,15 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
|
|||
raise RuntimeError(f'Failed to extract readability for {url} got {result.return_code}') from e
|
||||
|
||||
|
||||
async def singlefile_to_archive(singlefile: bytes) -> Archive:
|
||||
async def singlefile_to_archive(singlefile: bytes, destination: pathlib.Path = None) -> Archive:
|
||||
"""
|
||||
Convert a SingleFile to an Archive.
|
||||
|
||||
This is done by extracting readability, creating a screenshot, then attaching them to an Archive/FileGroup.
|
||||
|
||||
Args:
|
||||
singlefile: The SingleFile HTML bytes
|
||||
destination: Optional directory to save files to. If provided, files go here instead of archive/<domain>
|
||||
"""
|
||||
# Get URL first because it does some simple checking of `singlefile`
|
||||
url = get_url_from_singlefile(singlefile)
|
||||
|
|
@ -1146,7 +1168,7 @@ async def singlefile_to_archive(singlefile: bytes) -> Archive:
|
|||
logger.error(f'Failed to extract screenshot from: {url}', exc_info=e)
|
||||
|
||||
logger.trace(f'singlefile_to_archive modeling: {url}')
|
||||
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot)
|
||||
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot, destination=destination)
|
||||
return archive
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -138,3 +138,65 @@ async def test_model_archive_extracts_url_from_singlefile(async_client, test_ses
|
|||
collection = test_session.query(Collection).get(archive.collection_id)
|
||||
assert collection.name == domain
|
||||
assert collection.kind == 'domain'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_outside_directory_keeps_collection_directory(async_client, test_session, test_directory,
|
||||
make_files_structure):
|
||||
"""
|
||||
Test that model_archive() does NOT clear collection.directory when archive is outside it.
|
||||
|
||||
Bug: Previously, if an archive was found outside its domain collection's directory,
|
||||
the code would clear collection.directory = None. This was wrong because:
|
||||
1. Archives should still be associated with domain collection based on URL
|
||||
2. The collection's directory should remain intact for other archives
|
||||
"""
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
domain = 'zerohedge.com'
|
||||
url = f'https://{domain}/article'
|
||||
|
||||
# Create a domain collection WITH a specific directory
|
||||
collection_dir = 'archive/News/zerohedge.com'
|
||||
collection = Collection(name=domain, kind='domain', directory=collection_dir)
|
||||
test_session.add(collection)
|
||||
test_session.commit()
|
||||
collection_id = collection.id
|
||||
|
||||
# Create the collection directory
|
||||
(test_directory / collection_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create archive files in a DIFFERENT directory (simulating manual file move)
|
||||
old_archive_dir = test_directory / 'archive' / domain
|
||||
old_archive_dir.mkdir(parents=True)
|
||||
|
||||
timestamp = archive_strftime(datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC))
|
||||
title = 'Test Article'
|
||||
|
||||
singlefile_content = f'''<!DOCTYPE html><html><!--
|
||||
Page saved with SingleFile
|
||||
url: {url}
|
||||
--><head><title>{title}</title></head>
|
||||
<body><p>Content</p></body></html>'''
|
||||
|
||||
files = make_files_structure({
|
||||
str(old_archive_dir / f'{timestamp}_{title}.html'): singlefile_content.strip(),
|
||||
str(old_archive_dir / f'{timestamp}_{title}.readability.json'): '{"title": "' + title + '"}',
|
||||
})
|
||||
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
file_paths = [pathlib.Path(f) for f in files]
|
||||
file_group = FileGroup.from_paths(test_session, *file_paths)
|
||||
|
||||
# Model the archive - archive is in archive/zerohedge.com but collection dir is archive/News/zerohedge.com
|
||||
archive = model_archive(test_session, file_group)
|
||||
|
||||
assert archive is not None
|
||||
assert archive.collection_id == collection_id, "Archive should be associated with domain collection"
|
||||
|
||||
# CRITICAL: The collection's directory should NOT be cleared
|
||||
test_session.refresh(collection)
|
||||
assert str(collection.directory) == collection_dir, \
|
||||
f"Collection directory should NOT be cleared! Expected '{collection_dir}', got '{collection.directory}'"
|
||||
|
|
|
|||
|
|
@ -393,6 +393,25 @@ def test_get_new_archive_files(fake_now):
|
|||
assert str(archive_files.screenshot).endswith('archive/example.com/2001-01-01-00-00-00_Title.png')
|
||||
|
||||
|
||||
@skip_circleci
|
||||
def test_get_new_archive_files_with_destination(test_directory, fake_now):
|
||||
"""Archive files can be created in a custom destination directory instead of the default."""
|
||||
fake_now(datetime(2001, 1, 1))
|
||||
custom_destination = test_directory / 'archive/News/custom-domain.com'
|
||||
custom_destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# When destination is provided, files should go there instead of archive/<domain>
|
||||
archive_files = get_new_archive_files('https://example.com/page', 'Title', destination=custom_destination)
|
||||
|
||||
# Files should be in the custom destination, not archive/example.com
|
||||
assert str(archive_files.singlefile).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.html')
|
||||
assert str(archive_files.readability).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.html')
|
||||
assert str(archive_files.readability_txt).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.txt')
|
||||
assert str(archive_files.readability_json).endswith(
|
||||
'archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.json')
|
||||
assert str(archive_files.screenshot).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.png')
|
||||
|
||||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_title_in_filename(test_session, fake_now, test_directory, image_bytes_factory):
|
||||
|
|
|
|||
|
|
@ -1509,7 +1509,11 @@ class RSSDownloader(Downloader):
|
|||
next_download_settings['video_resolutions'] = i
|
||||
if i := settings.get('video_format'):
|
||||
next_download_settings['video_format'] = i
|
||||
if i := settings.get('destination'):
|
||||
# Use download.destination column (settings['destination'] is legacy)
|
||||
if download.destination:
|
||||
next_download_settings['destination'] = str(download.destination)
|
||||
elif i := settings.get('destination'):
|
||||
# Fallback to legacy settings for backwards compatibility
|
||||
next_download_settings['destination'] = i
|
||||
|
||||
result = DownloadResult(
|
||||
|
|
|
|||
|
|
@ -123,3 +123,58 @@ async def test_rss_downloader_filter_titles(test_session):
|
|||
dict(link='https://example.com/a', title='A'),
|
||||
dict(link='https://example.com/b', title='b'),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rss_downloader_uses_destination_column(test_session, test_download_manager, await_switches):
|
||||
"""RSS downloader should pass download.destination to child downloads, not settings['destination'].
|
||||
|
||||
The destination column is the source of truth. settings['destination'] is legacy.
|
||||
"""
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
http_downloader = RSSHTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
mock_parse_feed.return_value = dict(
|
||||
bozo=0,
|
||||
entries=[
|
||||
dict(link='https://example.com/a'),
|
||||
dict(link='https://example.com/b'),
|
||||
]
|
||||
)
|
||||
# Create RSS download with destination column set, but empty settings.
|
||||
test_download_manager.create_download(
|
||||
test_session,
|
||||
'https://example.com/feed',
|
||||
rss_downloader.name,
|
||||
sub_downloader_name='rss_http',
|
||||
destination='archive/custom/path', # destination column
|
||||
settings={}, # Empty settings - destination should come from column
|
||||
)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
# Get all downloads
|
||||
downloads = test_session.query(Download).order_by(Download.id).all()
|
||||
assert len(downloads) == 3, 'Expected 1 RSS download + 2 child downloads'
|
||||
|
||||
# The RSS download itself
|
||||
rss_download = downloads[0]
|
||||
assert rss_download.url == 'https://example.com/feed'
|
||||
assert str(rss_download.destination).endswith('archive/custom/path')
|
||||
|
||||
# Child downloads should have destination in their settings (passed from RSS download.destination)
|
||||
child_a = downloads[1]
|
||||
child_b = downloads[2]
|
||||
assert child_a.url == 'https://example.com/a'
|
||||
assert child_b.url == 'https://example.com/b'
|
||||
# The destination should be passed from RSS download.destination column to child settings
|
||||
assert child_a.settings.get('destination') is not None, \
|
||||
'Child download should inherit destination from RSS download.destination column'
|
||||
assert child_a.settings['destination'].endswith('archive/custom/path'), \
|
||||
f"Expected destination ending with 'archive/custom/path', got {child_a.settings.get('destination')}"
|
||||
assert child_b.settings.get('destination') is not None, \
|
||||
'Child download should inherit destination from RSS download.destination column'
|
||||
assert child_b.settings['destination'].endswith('archive/custom/path'), \
|
||||
f"Expected destination ending with 'archive/custom/path', got {child_b.settings.get('destination')}"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue