diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 4db00f9f..5101f331 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1565,6 +1565,13 @@ class RSSDownloader(Downloader): else: logger.warning(f'RSS entry {idx} did not have a link!') + if download.sub_downloader == 'video': + # Normalize video URLs before checking if already downloaded. + # YouTube Shorts URLs (e.g., youtube.com/shorts/ABC123) must be converted to watch URLs + # (e.g., youtube.com/watch?v=ABC123) to match what is stored in the database. + from modules.videos.normalize_video_url import normalize_video_url + urls = [normalize_video_url(i) for i in urls] + # Only download new URLs. with get_db_session() as session: urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(session, *urls)]] diff --git a/wrolpi/test/test_rss.py b/wrolpi/test/test_rss.py index 021b9401..39116232 100644 --- a/wrolpi/test/test_rss.py +++ b/wrolpi/test/test_rss.py @@ -5,7 +5,7 @@ import mock import pytest from sqlalchemy.orm import Session -from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadManager, RSSDownloader +from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadManager, RSSDownloader, download_manager class RSSHTTPDownloader(Downloader): @@ -178,3 +178,119 @@ async def test_rss_downloader_uses_destination_column(test_session, test_downloa 'Child download should inherit destination from RSS download.destination column' assert child_b.settings['destination'].endswith('archive/custom/path'), \ f"Expected destination ending with 'archive/custom/path', got {child_b.settings.get('destination')}" + + +@pytest.mark.asyncio +async def test_rss_downloader_normalizes_youtube_shorts_urls(test_session, test_download_manager, await_switches, + video_factory): + """RSS downloader should normalize YouTube Shorts URLs before checking if already downloaded. + + YouTube Shorts URLs (youtube.com/shorts/ABC123) are stored as watch URLs (youtube.com/watch?v=ABC123) + in the database after download. The RSS feed may continue to return shorts URLs, so normalization + is needed to recognize that the video was already downloaded. + """ + from modules.videos.downloader import VideoDownloader + from wrolpi.files.models import FileGroup + + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + video_downloader = VideoDownloader() + test_download_manager.register_downloader(video_downloader) + + # Create a video that was already downloaded. The URL is stored in the normalized watch format + # (as yt-dlp would store it after downloading). + video = video_factory(source_id='ABC123') + video.file_group.url = 'https://www.youtube.com/watch?v=ABC123' # Normalized URL + test_session.commit() + + # Verify the video exists with the normalized URL. + file_groups = list(test_session.query(FileGroup).filter( + FileGroup.url == 'https://www.youtube.com/watch?v=ABC123' + )) + assert len(file_groups) == 1, 'Video should exist with normalized URL' + + with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: + # RSS feed returns the shorts URL format (same video, different URL format). + mock_parse_feed.return_value = dict( + bozo=0, + entries=[ + dict(link='https://www.youtube.com/shorts/ABC123'), # Shorts URL, same video + dict(link='https://www.youtube.com/shorts/DEF456'), # New video, not downloaded + ] + ) + test_download_manager.create_download( + test_session, + 'https://example.com/feed', + rss_downloader.name, + sub_downloader_name='video', + ) + await test_download_manager.wait_for_all_downloads() + + # Get all downloads. + downloads = test_session.query(Download).order_by(Download.id).all() + + # Should have: + # 1. RSS feed download (complete) + # 2. Only the new video (DEF456) should be added for download + # The already downloaded video (ABC123) should NOT be re-added + download_urls = [d.url for d in downloads] + assert 'https://example.com/feed' in download_urls, 'RSS feed download should exist' + + # The shorts URL for ABC123 should NOT be in downloads (it was recognized as already downloaded). + assert 'https://www.youtube.com/shorts/ABC123' not in download_urls, \ + 'Already downloaded video (shorts URL) should not be re-added to downloads' + assert 'https://www.youtube.com/watch?v=ABC123' not in download_urls, \ + 'Already downloaded video (normalized URL) should not be re-added to downloads' + + # The new video (DEF456) should be added for download with the normalized URL. + assert 'https://www.youtube.com/watch?v=DEF456' in download_urls, \ + 'New video should be added to downloads with normalized URL' + + +@pytest.mark.asyncio +async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_session, test_download_manager, await_switches): + """RSS downloader should normalize YouTube Shorts URLs before checking the skip list. + + If a video was skipped with a normalized URL, the RSS feed returning a shorts URL should + still recognize it as skipped. + """ + from modules.videos.downloader import VideoDownloader + + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + video_downloader = VideoDownloader() + test_download_manager.register_downloader(video_downloader) + + # Add a video to the skip list using the normalized URL format. + download_manager.add_to_skip_list('https://www.youtube.com/watch?v=SKIPPED123') + + with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: + # RSS feed returns the shorts URL format (same video, different URL format). + mock_parse_feed.return_value = dict( + bozo=0, + entries=[ + dict(link='https://www.youtube.com/shorts/SKIPPED123'), # Shorts URL, should be skipped + dict(link='https://www.youtube.com/shorts/NEW456'), # New video, should be downloaded + ] + ) + test_download_manager.create_download( + test_session, + 'https://example.com/feed', + rss_downloader.name, + sub_downloader_name='video', + ) + await test_download_manager.wait_for_all_downloads() + + # Get all downloads. + downloads = test_session.query(Download).order_by(Download.id).all() + download_urls = [d.url for d in downloads] + + # The skipped video should NOT be in downloads. + assert 'https://www.youtube.com/shorts/SKIPPED123' not in download_urls, \ + 'Skipped video (shorts URL) should not be added to downloads' + assert 'https://www.youtube.com/watch?v=SKIPPED123' not in download_urls, \ + 'Skipped video (normalized URL) should not be added to downloads' + + # The new video should be added for download with the normalized URL. + assert 'https://www.youtube.com/watch?v=NEW456' in download_urls, \ + 'New video should be added to downloads with normalized URL'