Storing parent_download_url in a file group's json file after download has completed.

modules/archive/__init__.py

@@ -69,6 +69,8 @@ class ArchiveDownloader(Downloader, ABC):
             archive = await lib.singlefile_to_archive(singlefile, destination=destination)
             archive_id = archive.id
 
+        settings = download.settings or dict()
+
         with get_db_session() as session:
             archive = Archive.find_by_id(session, archive_id)
             need_commit = False
@@ -80,6 +82,12 @@ class ArchiveDownloader(Downloader, ABC):
                 if need_commit:
                     session.commit()
 
+            if parent_download_url := settings.get('parent_download_url'):
+                try:
+                    archive.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
+                except ValueError as e:
+                    logger.warning(f'Failed to update wrolpi json for {archive}: {e}')
+
             logger.info(f'Successfully downloaded Archive {download.url} {archive}')
 
             return DownloadResult(success=True, location=f'/archive/{archive.id}')

modules/archive/test/test_models.py

@@ -1,11 +1,13 @@
+import json
 import pathlib
 from datetime import datetime
 
 import pytest
 
 from modules.archive import Archive
-from wrolpi.common import get_wrolpi_config
+from wrolpi.common import get_media_directory, get_wrolpi_config
 from wrolpi.errors import UnknownArchive
+from wrolpi.files.models import FileGroup
 
 
 def test_archive_get_by_id(test_session, archive_factory):
@@ -178,3 +180,42 @@ async def test_archive_validate_rebuilds_data_after_filename_change(async_client
     archive.validate()
 
     assert archive.file_group.data['singlefile_path'] == new_filename
+
+
+def test_update_wrolpi_json_prefers_info_json(test_session, make_files_structure):
+    """FileGroup.update_wrolpi_json prefers .info.json over .readability.json when both exist."""
+    make_files_structure({'page.html': '<html>test</html>'})
+    fg = FileGroup.from_paths(test_session, get_media_directory() / 'page.html')
+    test_session.add(fg)
+    test_session.flush()
+
+    fg.replace_info_json({'title': 'Info Title'})
+    fg.replace_readability_json({'title': 'Readability Title'})
+
+    fg.update_wrolpi_json({'key': 'value'})
+
+    # wrolpi section should be in .info.json, not .readability.json.
+    info_json = json.loads((get_media_directory() / 'page.info.json').read_text())
+    assert info_json['wrolpi'] == {'key': 'value'}
+
+    readability_json = json.loads((get_media_directory() / 'page.readability.json').read_text())
+    assert 'wrolpi' not in readability_json
+
+
+def test_metadata_json_path(test_session, make_files_structure):
+    """FileGroup.metadata_json_path returns .info.json first, then .readability.json."""
+    make_files_structure({'page.html': '<html>test</html>'})
+    fg = FileGroup.from_paths(test_session, get_media_directory() / 'page.html')
+    test_session.add(fg)
+    test_session.flush()
+
+    # No metadata JSON yet.
+    assert fg.metadata_json_path is None
+
+    # Add readability JSON.
+    fg.replace_readability_json({'title': 'Test'})
+    assert fg.metadata_json_path == get_media_directory() / 'page.readability.json'
+
+    # Add info JSON - it should now be preferred.
+    fg.replace_info_json({'title': 'Test'})
+    assert fg.metadata_json_path == get_media_directory() / 'page.info.json'

modules/videos/downloader.py

@@ -672,6 +672,12 @@ class VideoDownloader(Downloader, ABC):
                 if video_info_json_path and (new_info_json := video.clean_info_json()):
                     video.replace_info_json(new_info_json, clean=False)
 
+                if parent_download_url := settings.get('parent_download_url'):
+                    try:
+                        video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
+                    except ValueError as e:
+                        logger.warning(f'Failed to update wrolpi json for {video}: {e}')
+
                 # Update channel's collection's file_format to track the format used
                 if channel_id:
                     from modules.videos.models import Channel
@@ -903,11 +909,19 @@ class VideoDownloader(Downloader, ABC):
             logger.error('Failed to download video info json', exc_info=e)
             return DownloadResult(success=False, error='Failed to download video info json')
 
+        settings = download.settings or dict()
+
         with get_db_session(commit=True) as session:
             # Find the existing video, replace its info json.
             video = Video.get_by_url(url, session)
             video.replace_info_json(info_json)
 
+            if parent_download_url := settings.get('parent_download_url'):
+                try:
+                    video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
+                except ValueError as e:
+                    logger.warning(f'Failed to update wrolpi json for {video}: {e}')
+
             if video.get_comments():
                 video.have_comments = True
 

modules/videos/models.py

@@ -10,7 +10,7 @@ from sqlalchemy.orm.collections import InstrumentedList
 from modules.videos.errors import UnknownVideo, UnknownChannel
 from wrolpi.captions import read_captions
 from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
-    background_task, replace_file
+    background_task
 from wrolpi.db import get_db_curs, get_db_session
 from wrolpi.downloader import Download
 from wrolpi.files.lib import split_path_stem_and_suffix
@@ -546,19 +546,10 @@ class Video(ModelHelper, Base):
 
     def replace_info_json(self, info_json: dict, clean: bool = True, format_: bool = True):
         """Replace the info json file with the new json dict.  Handles adding new info_json file, if necessary."""
-        info_json_path = self.info_json_path or self.video_path.with_suffix('.info.json')
         if clean:
             info_json = self.clean_info_json(info_json)
 
-        if format_:
-            info_json = json.dumps(info_json, indent=2, sort_keys=True)
-        else:
-            info_json = json.dumps(info_json)
-
-        replace_file(info_json_path, info_json, missing_ok=True)
-
-        if not self.info_json_path:
-            self.file_group.append_files(info_json_path)
+        self.file_group.replace_info_json(info_json, format_=format_)
 
 
 class Channel(ModelHelper, Base):

modules/videos/test/test_models.py

@@ -1,7 +1,10 @@
+import json
+
 import pytest
 
 from modules.videos.models import Video
-from wrolpi.common import get_relative_to_media_directory
+from wrolpi.common import get_media_directory, get_relative_to_media_directory
+from wrolpi.files.models import FileGroup
 
 
 @pytest.mark.asyncio()
@@ -47,3 +50,51 @@ async def test_channel_move(async_client, test_session, test_directory, channel_
     assert not extra_file.is_file()
     assert (test_directory / 'foo/New Channel Directory/extra.txt').is_file()
     assert (test_directory / 'foo/New Channel Directory/extra.txt').read_text() == 'extra stuff'
+
+
+def test_replace_info_json_compact(test_session, make_files_structure):
+    """FileGroup.replace_info_json can write compact JSON."""
+    make_files_structure({'video.mp4': 'fake video'})
+    fg = FileGroup.from_paths(test_session, get_media_directory() / 'video.mp4')
+    test_session.add(fg)
+    test_session.flush()
+
+    info = {'title': 'Test', 'id': '123'}
+    fg.replace_info_json(info, format_=False)
+
+    json_path = get_media_directory() / 'video.info.json'
+    content = json_path.read_text()
+    assert '\n' not in content
+    assert json.loads(content) == info
+
+
+def test_replace_info_json_multiple_json_files(test_session, make_files_structure):
+    """FileGroup.replace_info_json raises ValueError when multiple .info.json files are tracked."""
+    make_files_structure({
+        'video.mp4': 'fake video',
+        'video.info.json': '{"title": "one"}',
+        'video.other.info.json': '{"title": "two"}',
+    })
+    fg = FileGroup.from_paths(test_session, get_media_directory() / 'video.mp4')
+    test_session.add(fg)
+    test_session.flush()
+
+    fg.append_files(get_media_directory() / 'video.info.json', get_media_directory() / 'video.other.info.json')
+
+    with pytest.raises(ValueError, match='multiple .info.json files'):
+        fg.replace_info_json({'title': 'new'})
+
+
+def test_update_wrolpi_json_merges(test_session, make_files_structure):
+    """FileGroup.update_wrolpi_json merges new keys without clobbering existing ones."""
+    make_files_structure({'video.mp4': 'fake video'})
+    fg = FileGroup.from_paths(test_session, get_media_directory() / 'video.mp4')
+    test_session.add(fg)
+    test_session.flush()
+
+    fg.replace_info_json({'title': 'Test', 'wrolpi': {'existing_key': 'keep'}})
+    fg.update_wrolpi_json({'new_key': 'new_value'})
+
+    json_path = get_media_directory() / 'video.info.json'
+    written = json.loads(json_path.read_text())
+    assert written['wrolpi'] == {'existing_key': 'keep', 'new_key': 'new_value'}