From a8ff1a1c38b0c655165bbe66c3a8e91e0ece7a33 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 17 Dec 2025 16:56:58 -0700 Subject: [PATCH] Updating archive downloader to use download.destination. --- modules/archive/__init__.py | 34 ++++------ modules/archive/lib.py | 36 ++++++++--- .../archive/test/test_collection_id_bug.py | 62 +++++++++++++++++++ modules/archive/test/test_lib.py | 19 ++++++ wrolpi/downloader.py | 6 +- wrolpi/test/test_rss.py | 55 ++++++++++++++++ 6 files changed, 181 insertions(+), 31 deletions(-) diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 4a546f94..0868a79d 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -44,15 +44,21 @@ class ArchiveDownloader(Downloader, ABC): if download.attempts > 3: raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}') + # Get destination from settings (passed by RSS downloader from download.destination column) + destination = None + if download.settings and download.settings.get('destination'): + destination = pathlib.Path(download.settings['destination']) + if DOCKERIZED or PYTEST: # Perform the archive in the Archive docker container. (Typically in the development environment). singlefile, readability, screenshot = await request_archive(download.url) - archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot) + archive: Archive = await lib.model_archive_result( + download.url, singlefile, readability, screenshot, destination=destination) archive_id = archive.id else: # Perform the archive using locally installed executables. singlefile = await self.do_singlefile(download) - archive = await lib.singlefile_to_archive(singlefile) + archive = await lib.singlefile_to_archive(singlefile, destination=destination) archive_id = archive.id with get_db_session() as session: @@ -211,27 +217,9 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive: archive.validate() archive.flush() - # Validate that archive path matches domain directory (efficient O(1) check) - if archive.collection_id and archive.collection: - collection = archive.collection - if collection.kind == 'domain' and collection.directory and not collection.tag_id: - # Check if archive path is under the domain's directory - archive_path = pathlib.Path(file_group.primary_path) - from wrolpi.common import get_media_directory - media_dir = get_media_directory() - collection_abs_dir = media_dir / collection.directory - - try: - # Check if archive is under the collection's directory - archive_path.relative_to(collection_abs_dir) - except ValueError: - # Archive is NOT under the domain's directory - clear it - logger.warning( - f'Archive {archive.id} at {archive_path} is not under domain directory {collection.directory}. ' - f'Clearing domain directory (domain is not tagged).' - ) - collection.directory = None - session.flush([collection]) + # Note: We intentionally do NOT clear collection.directory when archive is outside it. + # Archives are associated with domain collections based on URL, not file location. + # The collection's directory should remain intact for other archives. file_group.title = file_group.a_text = title or archive.file_group.title file_group.d_text = contents diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 1d104391..ce9fd4ce 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -277,9 +277,19 @@ def get_archive_destination(domain_collection: 'Collection') -> pathlib.Path: MAXIMUM_ARCHIVE_FILE_CHARACTER_LENGTH = 200 -def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles: - """Create a list of archive files using a shared name schema. Raise an error if any of them exist.""" - directory = get_domain_directory(url) +def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.Path = None) -> ArchiveFiles: + """Create a list of archive files using a shared name schema. Raise an error if any of them exist. + + Args: + url: The URL being archived (used to derive domain if no destination) + title: Title for the archive files + destination: Optional directory to save files to. If provided, files go here instead of archive/ + """ + if destination: + directory = destination + directory.mkdir(parents=True, exist_ok=True) + else: + directory = get_domain_directory(url) # Datetime is valid in Linux and Windows. dt = archive_strftime(now()) @@ -391,9 +401,17 @@ async def request_screenshot(url: str, singlefile_path: pathlib.Path) -> Optiona return screenshot -async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes) -> Archive: +async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes, + destination: pathlib.Path = None) -> Archive: """ Convert results from ArchiveDownloader into real files. Create Archive record. + + Args: + url: The URL that was archived + singlefile: The HTML content from SingleFile + readability: The readability extraction results + screenshot: The screenshot bytes + destination: Optional directory to save files to. If provided, files go here instead of archive/ """ readability = readability.copy() if readability else readability # First try to get the title from Readability. @@ -406,7 +424,7 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr # Readability could not find title, lets use ours. readability['title'] = title - archive_files = get_new_archive_files(url, title) + archive_files = get_new_archive_files(url, title, destination=destination) if readability: # Write the readability parts to their own files. Write what is left after pops to the JSON file. @@ -1114,11 +1132,15 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120): raise RuntimeError(f'Failed to extract readability for {url} got {result.return_code}') from e -async def singlefile_to_archive(singlefile: bytes) -> Archive: +async def singlefile_to_archive(singlefile: bytes, destination: pathlib.Path = None) -> Archive: """ Convert a SingleFile to an Archive. This is done by extracting readability, creating a screenshot, then attaching them to an Archive/FileGroup. + + Args: + singlefile: The SingleFile HTML bytes + destination: Optional directory to save files to. If provided, files go here instead of archive/ """ # Get URL first because it does some simple checking of `singlefile` url = get_url_from_singlefile(singlefile) @@ -1146,7 +1168,7 @@ async def singlefile_to_archive(singlefile: bytes) -> Archive: logger.error(f'Failed to extract screenshot from: {url}', exc_info=e) logger.trace(f'singlefile_to_archive modeling: {url}') - archive: Archive = await model_archive_result(url, singlefile, readability, screenshot) + archive: Archive = await model_archive_result(url, singlefile, readability, screenshot, destination=destination) return archive diff --git a/modules/archive/test/test_collection_id_bug.py b/modules/archive/test/test_collection_id_bug.py index 73e657b4..02cfae64 100644 --- a/modules/archive/test/test_collection_id_bug.py +++ b/modules/archive/test/test_collection_id_bug.py @@ -138,3 +138,65 @@ async def test_model_archive_extracts_url_from_singlefile(async_client, test_ses collection = test_session.query(Collection).get(archive.collection_id) assert collection.name == domain assert collection.kind == 'domain' + + +@pytest.mark.asyncio +async def test_archive_outside_directory_keeps_collection_directory(async_client, test_session, test_directory, + make_files_structure): + """ + Test that model_archive() does NOT clear collection.directory when archive is outside it. + + Bug: Previously, if an archive was found outside its domain collection's directory, + the code would clear collection.directory = None. This was wrong because: + 1. Archives should still be associated with domain collection based on URL + 2. The collection's directory should remain intact for other archives + """ + import datetime + import pytz + + domain = 'zerohedge.com' + url = f'https://{domain}/article' + + # Create a domain collection WITH a specific directory + collection_dir = 'archive/News/zerohedge.com' + collection = Collection(name=domain, kind='domain', directory=collection_dir) + test_session.add(collection) + test_session.commit() + collection_id = collection.id + + # Create the collection directory + (test_directory / collection_dir).mkdir(parents=True, exist_ok=True) + + # Create archive files in a DIFFERENT directory (simulating manual file move) + old_archive_dir = test_directory / 'archive' / domain + old_archive_dir.mkdir(parents=True) + + timestamp = archive_strftime(datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC)) + title = 'Test Article' + + singlefile_content = f'''{title} +

Content

''' + + files = make_files_structure({ + str(old_archive_dir / f'{timestamp}_{title}.html'): singlefile_content.strip(), + str(old_archive_dir / f'{timestamp}_{title}.readability.json'): '{"title": "' + title + '"}', + }) + + from wrolpi.files.models import FileGroup + + file_paths = [pathlib.Path(f) for f in files] + file_group = FileGroup.from_paths(test_session, *file_paths) + + # Model the archive - archive is in archive/zerohedge.com but collection dir is archive/News/zerohedge.com + archive = model_archive(test_session, file_group) + + assert archive is not None + assert archive.collection_id == collection_id, "Archive should be associated with domain collection" + + # CRITICAL: The collection's directory should NOT be cleared + test_session.refresh(collection) + assert str(collection.directory) == collection_dir, \ + f"Collection directory should NOT be cleared! Expected '{collection_dir}', got '{collection.directory}'" diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 515a0eb1..8ef64b83 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -393,6 +393,25 @@ def test_get_new_archive_files(fake_now): assert str(archive_files.screenshot).endswith('archive/example.com/2001-01-01-00-00-00_Title.png') +@skip_circleci +def test_get_new_archive_files_with_destination(test_directory, fake_now): + """Archive files can be created in a custom destination directory instead of the default.""" + fake_now(datetime(2001, 1, 1)) + custom_destination = test_directory / 'archive/News/custom-domain.com' + custom_destination.mkdir(parents=True, exist_ok=True) + + # When destination is provided, files should go there instead of archive/ + archive_files = get_new_archive_files('https://example.com/page', 'Title', destination=custom_destination) + + # Files should be in the custom destination, not archive/example.com + assert str(archive_files.singlefile).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.html') + assert str(archive_files.readability).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.html') + assert str(archive_files.readability_txt).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.txt') + assert str(archive_files.readability_json).endswith( + 'archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.json') + assert str(archive_files.screenshot).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.png') + + @skip_circleci @pytest.mark.asyncio async def test_title_in_filename(test_session, fake_now, test_directory, image_bytes_factory): diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index a7b3fc9b..5f16ee96 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1509,7 +1509,11 @@ class RSSDownloader(Downloader): next_download_settings['video_resolutions'] = i if i := settings.get('video_format'): next_download_settings['video_format'] = i - if i := settings.get('destination'): + # Use download.destination column (settings['destination'] is legacy) + if download.destination: + next_download_settings['destination'] = str(download.destination) + elif i := settings.get('destination'): + # Fallback to legacy settings for backwards compatibility next_download_settings['destination'] = i result = DownloadResult( diff --git a/wrolpi/test/test_rss.py b/wrolpi/test/test_rss.py index 27ea48cd..021b9401 100644 --- a/wrolpi/test/test_rss.py +++ b/wrolpi/test/test_rss.py @@ -123,3 +123,58 @@ async def test_rss_downloader_filter_titles(test_session): dict(link='https://example.com/a', title='A'), dict(link='https://example.com/b', title='b'), ] + + +@pytest.mark.asyncio +async def test_rss_downloader_uses_destination_column(test_session, test_download_manager, await_switches): + """RSS downloader should pass download.destination to child downloads, not settings['destination']. + + The destination column is the source of truth. settings['destination'] is legacy. + """ + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + http_downloader = RSSHTTPDownloader() + test_download_manager.register_downloader(http_downloader) + + with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: + mock_parse_feed.return_value = dict( + bozo=0, + entries=[ + dict(link='https://example.com/a'), + dict(link='https://example.com/b'), + ] + ) + # Create RSS download with destination column set, but empty settings. + test_download_manager.create_download( + test_session, + 'https://example.com/feed', + rss_downloader.name, + sub_downloader_name='rss_http', + destination='archive/custom/path', # destination column + settings={}, # Empty settings - destination should come from column + ) + await test_download_manager.wait_for_all_downloads() + + # Get all downloads + downloads = test_session.query(Download).order_by(Download.id).all() + assert len(downloads) == 3, 'Expected 1 RSS download + 2 child downloads' + + # The RSS download itself + rss_download = downloads[0] + assert rss_download.url == 'https://example.com/feed' + assert str(rss_download.destination).endswith('archive/custom/path') + + # Child downloads should have destination in their settings (passed from RSS download.destination) + child_a = downloads[1] + child_b = downloads[2] + assert child_a.url == 'https://example.com/a' + assert child_b.url == 'https://example.com/b' + # The destination should be passed from RSS download.destination column to child settings + assert child_a.settings.get('destination') is not None, \ + 'Child download should inherit destination from RSS download.destination column' + assert child_a.settings['destination'].endswith('archive/custom/path'), \ + f"Expected destination ending with 'archive/custom/path', got {child_a.settings.get('destination')}" + assert child_b.settings.get('destination') is not None, \ + 'Child download should inherit destination from RSS download.destination column' + assert child_b.settings['destination'].endswith('archive/custom/path'), \ + f"Expected destination ending with 'archive/custom/path', got {child_b.settings.get('destination')}"